aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorLukasz Kasprzak <lukasz.kasprzak@pm.me>2026-03-19 15:05:32 +0100
committerLukasz Kasprzak <lukasz.kasprzak@pm.me>2026-03-19 15:05:32 +0100
commitb1844d805a8f190af00faf3f6e5bed7d997ecaae (patch)
treed6296ba5e85034836e203a542d2ad34a9e5c5b32
parentc74163f1ecbd809c83bb465ee89fa04369a89972 (diff)
downloadbin-b1844d805a8f190af00faf3f6e5bed7d997ecaae.tar.gz
bin-b1844d805a8f190af00faf3f6e5bed7d997ecaae.zip
created sorter for sorting downloads with custom rules; created straper for recreation of server
-rw-r--r--__pycache__/dumppdf.cpython-313.pycbin0 -> 19855 bytes
-rw-r--r--__pycache__/pdf2txt.cpython-313.pycbin0 -> 11249 bytes
-rw-r--r--docs/kvm-lab-readme.md162
m---------lib/bashsimplecurses0
-rwxr-xr-xmail-i2p-fetch2
-rwxr-xr-xmailodf95
-rwxr-xr-xmedia-toggle10
-rwxr-xr-xmutt-pdf5
-rwxr-xr-xofficium_curses.sh212
-rwxr-xr-xpdf-open6
-rwxr-xr-xpip-bins/csv2ods229
-rwxr-xr-xpip-bins/dumppdf.py468
-rwxr-xr-xpip-bins/odf2mht72
-rwxr-xr-xpip-bins/odf2xhtml59
-rwxr-xr-xpip-bins/odf2xml81
-rwxr-xr-xpip-bins/odfimgimport190
-rwxr-xr-xpip-bins/odflint216
-rwxr-xr-xpip-bins/odfmeta266
-rwxr-xr-xpip-bins/odfoutline144
-rwxr-xr-xpip-bins/odfuserfield101
-rwxr-xr-xpip-bins/pdf2txt.py324
-rwxr-xr-xpip-bins/pdfplumber8
-rwxr-xr-xpip-bins/pypdfium28
-rwxr-xr-xpip-bins/xml2odf241
-rwxr-xr-xsimplex-chatbin0 -> 73935832 bytes
-rw-r--r--sorter/readme.md210
-rw-r--r--sorter/rules.toml124
-rw-r--r--sorter/sort_downloads.py309
-rwxr-xr-xsorter/sort_downloads.sh60
-rw-r--r--straper/README.md123
-rw-r--r--straper/common.sh330
-rw-r--r--straper/doctor.sh222
-rw-r--r--straper/install-base.sh342
-rw-r--r--straper/rebuild.conf.example10
-rw-r--r--straper/restore-configs.sh521
-rw-r--r--straper/sanctum-rebuild-toolkit.tar.gzbin0 -> 14623 bytes
36 files changed, 5150 insertions, 0 deletions
diff --git a/__pycache__/dumppdf.cpython-313.pyc b/__pycache__/dumppdf.cpython-313.pyc
new file mode 100644
index 0000000..20a2388
--- /dev/null
+++ b/__pycache__/dumppdf.cpython-313.pyc
Binary files differ
diff --git a/__pycache__/pdf2txt.cpython-313.pyc b/__pycache__/pdf2txt.cpython-313.pyc
new file mode 100644
index 0000000..84e0609
--- /dev/null
+++ b/__pycache__/pdf2txt.cpython-313.pyc
Binary files differ
diff --git a/docs/kvm-lab-readme.md b/docs/kvm-lab-readme.md
new file mode 100644
index 0000000..89da6c1
--- /dev/null
+++ b/docs/kvm-lab-readme.md
@@ -0,0 +1,162 @@
+# `KVM Lab Workflow`
+
+**Purpose:**
+Fast creation of disposable or persistent KVM labs with strong isolation,
+no host contamination, and reversible state management.
+
+---
+
+## Directory Layout
+
+```
+/var/lib/libvirt/images/
+├── base/
+│ ├── debian13-template.qcow2
+│ └── work01-base-YYYY-MM-DD_HHMMSS.qcow2
+├── overlays/
+│ └── work01.qcow2
+├── lab/
+│ └── <lab-name>.qcow2
+├── whonix/
+│ ├── Whonix-Gateway-*.qcow2
+│ └── Whonix-Workstation-*.qcow2
+└── archive/
+```
+
+---
+
+## Installed Tools
+
+| Script | Location | Purpose |
+| --------------------- | ---------------------------------- | -------------------------------- |
+| `kvm-lab-create` | `~/.local/bin/kvm-lab-create` | Create lab VMs from chosen base |
+| `kvm-lab-destroy` | `~/.local/bin/kvm-lab-destroy` | Destroy lab VM + disk safely |
+| `kvm-promote-to-base` | `~/.local/bin/kvm-promote-to-base` | Freeze any VM into reusable base |
+| `kvm--lab-status` | `~/.local/bin/kvm-lab-status` | Prints the current status |
+Ensure:
+
+```
+echo $PATH | grep "$HOME/.local/bin"
+```
+
+---
+
+## Core Concepts
+
+| Term | Meaning |
+| ---------- | ------------------------------------------------ |
+| Base image | Immutable qcow2 used as parent for new labs |
+| Overlay | Writable qcow2 layer for a specific VM |
+| Promote | Freeze a VM’s current disk state into a new base |
+
+---
+
+## Normal Workflow
+
+### 1. Promote a VM into a base
+
+Freeze current VM state for reuse.
+
+```
+virsh -c qemu:///system shutdown work01
+kvm-promote-to-base work01
+```
+
+Creates:
+
+```
+/var/lib/libvirt/images/base/work01-base-YYYY-MM-DD_HHMMSS.qcow2
+```
+
+---
+
+### 2. Create a new lab
+
+From template:
+
+```
+kvm-lab-create lab10 --base template --start
+```
+
+From newest work01 base:
+
+```
+kvm-lab-create lab11 --base work01 --start
+```
+
+From specific base:
+
+```
+kvm-lab-create lab12 --base /var/lib/libvirt/images/base/work01-base-2026-01-09_150905.qcow2 --start
+```
+
+---
+
+### 3. Destroy a lab safely
+
+```
+kvm-lab-destroy lab11
+```
+
+**Guards enforced:**
+
+* Only deletes VMs whose disks are under `/var/lib/libvirt/images/lab/`
+* Refuses to touch `work01`, template, Whonix, or base images
+
+---
+
+## Debugging & Verification
+
+List all VMs:
+
+```
+virsh -c qemu:///system list --all
+```
+
+Check disk chain:
+
+```
+sudo qemu-img info --backing-chain /var/lib/libvirt/images/lab/lab10.qcow2
+```
+
+Confirm no base corruption:
+
+```
+sudo qemu-img info /var/lib/libvirt/images/base/work01-base-*.qcow2
+```
+
+---
+
+## Invariants (Never Broken)
+
+* `work01` remains intact
+* Base images are never modified
+* Lab VMs cannot access host or other VMs
+* No filesystem passthrough
+* All labs use NAT-only `default` network
+
+---
+
+## Recovery
+
+Delete a bad base:
+
+```
+sudo rm /var/lib/libvirt/images/base/<base-file>.qcow2
+```
+
+Delete a stuck lab:
+
+```
+kvm-lab-destroy <lab-name>
+```
+
+---
+
+## Philosophy
+
+> **One-way operations**
+> Promote → Create → Destroy
+> Host always stays clean.
+
+
diff --git a/lib/bashsimplecurses b/lib/bashsimplecurses
deleted file mode 160000
-Subproject 006b9cafd4d381b693c5e4f55140f2834cf3087
diff --git a/mail-i2p-fetch b/mail-i2p-fetch
new file mode 100755
index 0000000..7153cfb
--- /dev/null
+++ b/mail-i2p-fetch
@@ -0,0 +1,2 @@
+#!/bin/sh
+getmail --rcfile "$HOME/.config/getmail/i2p/getmailrc"
diff --git a/mailodf b/mailodf
new file mode 100755
index 0000000..0c5da54
--- /dev/null
+++ b/mailodf
@@ -0,0 +1,95 @@
+#!/usr/bin/python3
+# -*- coding: utf-8 -*-
+# Copyright (C) 2006 Søren Roug, European Environment Agency
+#
+# This is free software. You may redistribute it under the terms
+# of the Apache license and the GNU General Public License Version
+# 2 or at your option any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public
+# License along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+#
+# Contributor(s):
+#
+from odf.odf2xhtml import ODF2XHTML
+import zipfile
+import sys, os, smtplib, getopt
+
+from email.mime.multipart import MIMEMultipart
+from email.mime.nonmultipart import MIMENonMultipart
+from email.mime.text import MIMEText
+from email.encoders import encode_base64
+
+if sys.version_info[0]==3: unicode=str
+
+def usage():
+ sys.stderr.write("Usage: %s [-f from] [-s subject] inputfile recipients...\n" % sys.argv[0])
+
+try:
+ opts, args = getopt.getopt(sys.argv[1:], "f:s:", ["from=", "subject="])
+except getopt.GetoptError:
+ usage()
+ sys.exit(2)
+
+fromaddr = os.getlogin() + "@" + os.getenv('HOSTNAME','localhost')
+subject = None
+for o, a in opts:
+ if o in ("-f", "--from"):
+ fromaddr = a
+ if o in ("-s", "--subject"):
+ subject = a
+
+if len(args) < 2:
+ usage()
+ sys.exit(2)
+
+suffices = {
+ 'wmf':('image','x-wmf'),
+ 'png':('image','png'),
+ 'gif':('image','gif'),
+ 'jpg':('image','jpeg'),
+ 'jpeg':('image','jpeg')
+ }
+
+msg = MIMEMultipart('related',type="text/html")
+msg['From'] = fromaddr
+# msg['Date'] = strftime("%a, %d %b %Y %H:%M:%S +0000", gmtime())
+msg['To'] = ','.join(args[1:])
+msg.preamble = 'This is a multi-part message in MIME format.'
+msg.epilogue = ''
+odhandler = ODF2XHTML()
+result = odhandler.odf2xhtml(unicode(args[0]))
+if subject:
+ msg['Subject'] = subject
+else:
+ msg['Subject'] = odhandler.title
+htmlpart = MIMEText(result,'html','us-ascii')
+htmlpart['Content-Location'] = 'index.html'
+msg.attach(htmlpart)
+z = zipfile.ZipFile(unicode(args[0]))
+for file in z.namelist():
+ if file[0:9] == 'Pictures/':
+ suffix = file[file.rfind(".")+1:]
+ main,sub = suffices.get(suffix,('application','octet-stream'))
+ img = MIMENonMultipart(main,sub)
+ img.set_payload(z.read(file))
+ img['Content-Location'] = "" + file
+ encode_base64(img)
+ msg.attach(img)
+z.close()
+
+server = smtplib.SMTP('localhost')
+#server.set_debuglevel(1)
+server.sendmail(fromaddr, args[1:], msg.as_string())
+server.quit()
+
+
+# Local Variables: ***
+# mode: python ***
+# End: ***
diff --git a/media-toggle b/media-toggle
new file mode 100755
index 0000000..6aecc69
--- /dev/null
+++ b/media-toggle
@@ -0,0 +1,10 @@
+#!/bin/sh
+
+if mpc status | grep -q '^\[playing\]'; then
+ mpc pause
+ playerctl --all-players pause 2>/dev/null
+else
+ mpc play
+ playerctl --all-players play 2>/dev/null
+fi
+
diff --git a/mutt-pdf b/mutt-pdf
new file mode 100755
index 0000000..be0ab56
--- /dev/null
+++ b/mutt-pdf
@@ -0,0 +1,5 @@
+#!/bin/sh
+mkdir -p ~/Downloads/mail
+dest=~/Downloads/mail/mutt_$$.pdf
+cp "$1" "$dest"
+zathura "$dest"
diff --git a/officium_curses.sh b/officium_curses.sh
new file mode 100755
index 0000000..7afdd32
--- /dev/null
+++ b/officium_curses.sh
@@ -0,0 +1,212 @@
+#!/bin/bash
+# =============================================================================
+# officium_curses.sh — Liturgy of the Hours viewer using bashsimplecurses
+#
+# Full-screen TUI inspired by bashmount:
+#
+# ┌─ <liturgical day title> ──────────────────────────────────────────────┐
+# │ 1) Laudes 2) Prima 3) Tertia 4) Sexta 5) Nona 6) Vesperae │
+# └───────────────────────────────────────────────────────────────────────┘
+# ┌─ <Hora> ───────────────────────────────────────────────────────────────┐
+# │ Prayer text ... │
+# └───────────────────────────────────────────────────────────────────────┘
+# ┌─ Commands ─────────────────────────────────────────────────────────────┐
+# │ [1-6]: select hora [Enter]: refresh [q]: quit │
+# └───────────────────────────────────────────────────────────────────────┘
+# Command: _
+#
+# Authors : Adam Gomułka, Łukasz Kasprzak
+# Created : 2026-03-05
+# Version : 0.2
+# =============================================================================
+
+set -eo pipefail
+
+# ── Library ───────────────────────────────────────────────────────────────────
+
+BSC_LIB="${HOME}/.local/lib/bashsimplecurses/simple_curses.sh"
+
+if [ ! -f "$BSC_LIB" ]; then
+ echo "Error: bashsimplecurses not found at $BSC_LIB"
+ echo "Expected path: $BSC_LIB"
+ exit 1
+fi
+
+source "$BSC_LIB"
+
+# ── Configuration ─────────────────────────────────────────────────────────────
+
+BASE_URL="https://www.divinumofficium.com/cgi-bin/horas/officium.pl"
+HORAS=(Laudes Prima Tertia Sexta Nona Vesperae)
+
+# Word-wrap width: full terminal width minus window border/padding
+TERM_WIDTH=$(tput cols 2>/dev/null || echo 80)
+WRAP="${WRAP:-$((TERM_WIDTH - 6))}"
+
+# ── State ─────────────────────────────────────────────────────────────────────
+
+SELECTED=1 # currently selected hora (0-based index into HORAS)
+DIES="" # liturgical day title (from purple font tag)
+STATUS="" # shown in the commands bar
+declare -a PRAYER # prayer text, pre-split into lines for BSC
+
+HTML_TMP="$(mktemp)"
+trap 'rm -f "$HTML_TMP"' EXIT
+
+# ── Fetch and extract ─────────────────────────────────────────────────────────
+
+# Fetch the HTML for a given hora, extract the day title and prayer text,
+# and populate the DIES and PRAYER globals.
+fetch_hora() {
+ local hora="$1"
+ local url="${BASE_URL}?command=pray${hora}"
+
+ STATUS="Fetching ${hora}..."
+
+ if ! curl -s --max-time 30 "$url" -o "$HTML_TMP" 2>/dev/null; then
+ STATUS="Error: curl failed"
+ PRAYER=("Could not reach divinumofficium.com." "Check your network connection.")
+ DIES=""
+ return 1
+ fi
+ if [ ! -s "$HTML_TMP" ]; then
+ STATUS="Error: empty response"
+ PRAYER=("Server returned an empty response.")
+ DIES=""
+ return 1
+ fi
+
+ # Extract the liturgical day title from the purple font tag near the top
+ DIES=$(grep -o '<FONT COLOR="purple">[^<]*</FONT>' "$HTML_TMP" \
+ | sed 's/<[^>]*>//g; s/\r//')
+
+ # Extract and clean the Latin prayer text using the same pipeline as officium.sh
+ local raw_text
+ raw_text=$(awk '
+ /COLOR="red"><B><I>Incipit<\/I><\/B><\/FONT>/ { printing=1 }
+ printing {
+ if (/COLOR=.?green.?>[0-9]/) td_col=2
+ if (/<\/TD>/ && td_col==2) td_col=0
+ if (/<\/TR>/) { td_col=0; print "" }
+ if (td_col != 2) print
+ }
+ /<\/TABLE>/ && printing { exit }
+ ' "$HTML_TMP" \
+ | sed 's/<[^>]*>//g' \
+ | sed 's/\r$//' \
+ | sed 's/[[:space:]]*$//' \
+ | sed -E 's/^[0-9]+:[0-9]+[[:space:]]*//' \
+ | sed 's/&nbsp;/ /g' \
+ | sed 's/&ensp;/ /g' \
+ | sed 's/&lt;/</g' \
+ | sed 's/&gt;/>/g' \
+ | sed 's/&amp;/\&/g' \
+ | sed 's/&#[0-9]*;//g' \
+ | sed 's/&#x[0-9a-fA-F]*;//g' \
+ | sed -E 's/^(Psalmus[[:space:]]+[0-9]+)\([^)]*\)/\1/' \
+ | cat -s \
+ | grep -vE "^(Top[[:space:]]+Next|Top|Next)$" \
+ | grep -vE "^[A-Z][a-z]+ [A-Z][a-z]+\{" \
+ | grep -v "^[0-9]$")
+
+ # Word-wrap long lines and load into the PRAYER array.
+ # BSC's append takes one line at a time and does not wrap, so we must
+ # pre-wrap here before rendering.
+ PRAYER=()
+ while IFS= read -r line; do
+ if [ -z "$line" ]; then
+ PRAYER+=("")
+ elif [ "${#line}" -gt "$WRAP" ]; then
+ while IFS= read -r wrapped; do
+ PRAYER+=("$wrapped")
+ done < <(echo "$line" | fold -s -w "$WRAP")
+ else
+ PRAYER+=("$line")
+ fi
+ done <<< "$raw_text"
+
+ STATUS="$(date '+%A, %d %B %Y') — ${#PRAYER[@]} lines"
+}
+
+# ── BSC layout ────────────────────────────────────────────────────────────────
+
+main() {
+ local hora="${HORAS[$SELECTED]}"
+
+ # ── Window 1: Hora selector ──────────────────────────────────────────────
+ # Spans the full width; shows the liturgical day title and numbered hora list.
+
+ local win_title="Divinum Officium"
+ [ -n "$DIES" ] && win_title="$DIES"
+
+ window "$win_title" "red" "100%"
+ # Build one line of numbered hora buttons; selected hora is wrapped in []
+ local hora_line=" "
+ for i in "${!HORAS[@]}"; do
+ local n=$((i + 1))
+ if [ "$i" -eq "$SELECTED" ]; then
+ hora_line+="${n}) [${HORAS[$i]}] "
+ else
+ hora_line+="${n}) ${HORAS[$i]} "
+ fi
+ done
+ append "$hora_line"
+ endwin
+
+ # ── Window 2: Prayer text ────────────────────────────────────────────────
+
+ window "$hora" "blue" "100%"
+ for line in "${PRAYER[@]}"; do
+ # BSC collapses truly empty appends in some versions, so use a single
+ # space for blank lines to preserve paragraph spacing
+ if [ -z "$line" ]; then
+ append " "
+ else
+ append "$line"
+ fi
+ done
+ endwin
+
+ # ── Window 3: Commands ───────────────────────────────────────────────────
+
+ window "Commands" "green" "100%"
+ append " [1-6]: select hora [Enter]: refresh [q]: quit"
+ addsep
+ append " $STATUS"
+ endwin
+}
+
+# ── Interactive loop ──────────────────────────────────────────────────────────
+
+# Fetch the default hora (Prima) before showing the UI for the first time
+fetch_hora "${HORAS[$SELECTED]}"
+
+while true; do
+ # Render the current state using BSC
+ main_loop
+
+ # Print the command prompt below the BSC windows (outside any window frame)
+ printf "\nCommand: "
+
+ # Read one character without requiring Enter; -s suppresses echo.
+ # Timeout of 0.5s so the loop stays responsive.
+ IFS= read -r -s -n1 -t 0.5 key 2>/dev/null || true
+
+ case "$key" in
+ 1|2|3|4|5|6)
+ # Select hora by number (1-6) and fetch it
+ SELECTED=$(( key - 1 ))
+ fetch_hora "${HORAS[$SELECTED]}"
+ ;;
+ "")
+ # Enter (empty string from -n1) — refresh current hora
+ fetch_hora "${HORAS[$SELECTED]}"
+ ;;
+ q|Q)
+ clear
+ echo "Finis."
+ exit 0
+ ;;
+ # Any other key: just redraw without re-fetching
+ esac
+done
diff --git a/pdf-open b/pdf-open
new file mode 100755
index 0000000..ca5c427
--- /dev/null
+++ b/pdf-open
@@ -0,0 +1,6 @@
+#!/bin/bash
+echo "Got file: $1" >> /tmp/pdf-debug.log
+echo "File exists: $(test -f "$1" && echo yes || echo no)" >> /tmp/pdf-debug.log
+echo "File size: $(wc -c < "$1" 2>/dev/null)" >> /tmp/pdf-debug.log
+cp "$1" /tmp/mutt_pdf_$$.pdf
+zathura /tmp/mutt_pdf_$$.pdf
diff --git a/pip-bins/csv2ods b/pip-bins/csv2ods
new file mode 100755
index 0000000..c9ece9f
--- /dev/null
+++ b/pip-bins/csv2ods
@@ -0,0 +1,229 @@
+#!/usr/bin/python3
+# -*- coding: utf-8 -*-
+# Copyright (C) 2008 Agustin Henze -> agustinhenze at gmail.com
+#
+# This is free software. You may redistribute it under the terms
+# of the Apache license and the GNU General Public License Version
+# 2 or at your option any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public
+# License along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+#
+# Contributor(s):
+#
+# Søren Roug
+#
+# Oct 2014: Georges Khaznadar <georgesk@debian.org>
+# - ported to Python3
+# - imlemented the missing switch -c / --encoding, with an extra
+# feature for POSIX platforms which can guess encoding.
+
+from odf.opendocument import OpenDocumentSpreadsheet
+from odf.style import Style, TextProperties, ParagraphProperties, TableColumnProperties
+from odf.text import P
+from odf.table import Table, TableColumn, TableRow, TableCell
+from optparse import OptionParser
+import sys,csv,re, os, codecs
+
+if sys.version_info[0]==3: unicode=str
+
+if sys.version_info[0]==2:
+ class UTF8Recoder:
+ """
+ Iterator that reads an encoded stream and reencodes the input to UTF-8
+ """
+ def __init__(self, f, encoding):
+ self.reader = codecs.getreader(encoding)(f)
+
+ def __iter__(self):
+ return self
+
+ def next(self):
+ return self.reader.next().encode("utf-8")
+
+ class UnicodeReader:
+ """
+ A CSV reader which will iterate over lines in the CSV file "f",
+ which is encoded in the given encoding.
+ """
+
+ def __init__(self, f, dialect=csv.excel, encoding="utf-8", **kwds):
+ f = UTF8Recoder(f, encoding)
+ self.reader = csv.reader(f, dialect=dialect, **kwds)
+
+ def next(self):
+ row = self.reader.next()
+ return [unicode(s, "utf-8") for s in row]
+
+ def __iter__(self):
+ return self
+
+
+def csvToOds( pathFileCSV, pathFileODS, tableName='table',
+ delimiter=',', quoting=csv.QUOTE_MINIMAL,
+ quotechar = '"', escapechar = None,
+ skipinitialspace = False, lineterminator = '\r\n',
+ encoding="utf-8"):
+ textdoc = OpenDocumentSpreadsheet()
+ # Create a style for the table content. One we can modify
+ # later in the word processor.
+ tablecontents = Style(name="Table Contents", family="paragraph")
+ tablecontents.addElement(ParagraphProperties(numberlines="false", linenumber="0"))
+ tablecontents.addElement(TextProperties(fontweight="bold"))
+ textdoc.styles.addElement(tablecontents)
+
+ # Start the table
+ table = Table( name=tableName )
+
+ if sys.version_info[0]==3:
+ reader = csv.reader(open(pathFileCSV, encoding=encoding),
+ delimiter=delimiter,
+ quoting=quoting,
+ quotechar=quotechar,
+ escapechar=escapechar,
+ skipinitialspace=skipinitialspace,
+ lineterminator=lineterminator)
+ else:
+ reader = UnicodeReader(open(pathFileCSV),
+ encoding=encoding,
+ delimiter=delimiter,
+ quoting=quoting,
+ quotechar=quotechar,
+ escapechar=escapechar,
+ skipinitialspace=skipinitialspace,
+ lineterminator=lineterminator)
+ fltExp = re.compile('^\s*[-+]?\d+(\.\d+)?\s*$')
+
+ for row in reader:
+ tr = TableRow()
+ table.addElement(tr)
+ for val in row:
+ if fltExp.match(val):
+ tc = TableCell(valuetype="float", value=val.strip())
+ else:
+ tc = TableCell(valuetype="string")
+ tr.addElement(tc)
+ p = P(stylename=tablecontents,text=val)
+ tc.addElement(p)
+
+ textdoc.spreadsheet.addElement(table)
+ textdoc.save( pathFileODS )
+
+if __name__ == "__main__":
+ usage = "%prog -i file.csv -o file.ods -d"
+ parser = OptionParser(usage=usage, version="%prog 0.1")
+ parser.add_option('-i','--input', action='store',
+ dest='input', help='File input in csv')
+ parser.add_option('-o','--output', action='store',
+ dest='output', help='File output in ods')
+ parser.add_option('-d','--delimiter', action='store',
+ dest='delimiter', help='specifies a one-character string to use as the field separator. It defaults to ",".')
+
+ parser.add_option('-c','--encoding', action='store',
+ dest='encoding', help='specifies the encoding the file csv. It defaults to utf-8')
+
+ parser.add_option('-t','--table', action='store',
+ dest='tableName', help='The table name in the output file')
+
+ parser.add_option('-s','--skipinitialspace',
+ dest='skipinitialspace', help='''specifies how to interpret whitespace which
+ immediately follows a delimiter. It defaults to False, which
+ means that whitespace immediately following a delimiter is part
+ of the following field.''')
+
+ parser.add_option('-l','--lineterminator', action='store',
+ dest='lineterminator', help='''specifies the character sequence which should
+ terminate rows.''')
+
+ parser.add_option('-q','--quoting', action='store',
+ dest='quoting', help='''It can take on any of the following module constants:
+ 0 = QUOTE_MINIMAL means only when required, for example, when a field contains either the quotechar or the delimiter
+ 1 = QUOTE_ALL means that quotes are always placed around fields.
+ 2 = QUOTE_NONNUMERIC means that quotes are always placed around fields which do not parse as integers or floating point numbers.
+ 3 = QUOTE_NONE means that quotes are never placed around fields.
+ It defaults is QUOTE_MINIMAL''')
+
+ parser.add_option('-e','--escapechar', action='store',
+ dest='escapechar', help='''specifies a one-character string used to escape the delimiter when quoting is set to QUOTE_NONE.''')
+
+ parser.add_option('-r','--quotechar', action='store',
+ dest='quotechar', help='''specifies a one-character string to use as the quoting character. It defaults to ".''')
+
+ (options, args) = parser.parse_args()
+
+ if options.input:
+ pathFileCSV = options.input
+ else:
+ parser.print_help()
+ exit( 0 )
+
+ if options.output:
+ pathFileODS = options.output
+ else:
+ parser.print_help()
+ exit( 0 )
+
+ if options.delimiter:
+ delimiter = options.delimiter
+ else:
+ delimiter = ","
+
+ if options.skipinitialspace:
+ skipinitialspace = True
+ else:
+ skipinitialspace=False
+
+ if options.lineterminator:
+ lineterminator = options.lineterminator
+ else:
+ lineterminator ="\r\n"
+
+ if options.escapechar:
+ escapechar = options.escapechar
+ else:
+ escapechar=None
+
+ if options.tableName:
+ tableName = options.tableName
+ else:
+ tableName = "table"
+
+ if options.quotechar:
+ quotechar = options.quotechar
+ else:
+ quotechar = "\""
+
+ encoding = "utf-8" # default setting
+ ###########################################################
+ ## try to guess the encoding; this is implemented only with
+ ## POSIX platforms. Can it be improved?
+ output = os.popen('/usr/bin/file ' + pathFileCSV).read()
+ m=re.match(r'^.*: ([-a-zA-Z0-9]+) text$', output)
+ if m:
+ encoding=m.group(1)
+ if 'ISO-8859' in encoding:
+ encoding="latin-1"
+ else:
+ encoding="utf-8"
+ ############################################################
+ # when the -c or --coding switch is used, it takes precedence
+ if options.encoding:
+ encoding = options.encoding
+
+ csvToOds( pathFileCSV=unicode(pathFileCSV),
+ pathFileODS=unicode(pathFileODS),
+ delimiter=delimiter, skipinitialspace=skipinitialspace,
+ escapechar=escapechar,
+ lineterminator=unicode(lineterminator),
+ tableName=tableName, quotechar=quotechar,
+ encoding=encoding)
+
+# Local Variables: ***
+# mode: python ***
+# End: ***
diff --git a/pip-bins/dumppdf.py b/pip-bins/dumppdf.py
new file mode 100755
index 0000000..c2066aa
--- /dev/null
+++ b/pip-bins/dumppdf.py
@@ -0,0 +1,468 @@
+#!/usr/bin/python3
+"""Extract pdf structure in XML format"""
+
+import logging
+import os.path
+import re
+import sys
+from argparse import ArgumentParser
+from collections.abc import Container, Iterable
+from typing import Any, TextIO, cast
+
+import pdfminer
+from pdfminer.pdfdocument import PDFDocument, PDFNoOutlines, PDFXRefFallback
+from pdfminer.pdfexceptions import (
+ PDFIOError,
+ PDFObjectNotFound,
+ PDFTypeError,
+ PDFValueError,
+)
+from pdfminer.pdfpage import PDFPage
+from pdfminer.pdfparser import PDFParser
+from pdfminer.pdftypes import PDFObjRef, PDFStream, resolve1, stream_value
+from pdfminer.psparser import LIT, PSKeyword, PSLiteral
+from pdfminer.utils import isnumber
+
+logging.basicConfig()
+logger = logging.getLogger(__name__)
+
+ESC_PAT = re.compile(r'[\000-\037&<>()"\042\047\134\177-\377]')
+
+
+def escape(s: str | bytes) -> str:
+ us = str(s, "latin-1") if isinstance(s, bytes) else s
+ return ESC_PAT.sub(lambda m: f"&#{ord(m.group(0))};", us)
+
+
+def dumpxml(out: TextIO, obj: object, codec: str | None = None) -> None:
+ if obj is None:
+ out.write("<null />")
+ return
+
+ if isinstance(obj, dict):
+ out.write(f'<dict size="{len(obj)}">\n')
+ for k, v in obj.items():
+ out.write(f"<key>{k}</key>\n")
+ out.write("<value>")
+ dumpxml(out, v)
+ out.write("</value>\n")
+ out.write("</dict>")
+ return
+
+ if isinstance(obj, list):
+ out.write(f'<list size="{len(obj)}">\n')
+ for v in obj:
+ dumpxml(out, v)
+ out.write("\n")
+ out.write("</list>")
+ return
+
+ if isinstance(obj, (str, bytes)):
+ out.write(f'<string size="{len(obj)}">{escape(obj)}</string>')
+ return
+
+ if isinstance(obj, PDFStream):
+ if codec == "raw":
+ # Bug: writing bytes to text I/O. This will raise TypeError.
+ out.write(obj.get_rawdata()) # type: ignore [arg-type]
+ elif codec == "binary":
+ # Bug: writing bytes to text I/O. This will raise TypeError.
+ out.write(obj.get_data()) # type: ignore [arg-type]
+ else:
+ out.write("<stream>\n<props>\n")
+ dumpxml(out, obj.attrs)
+ out.write("\n</props>\n")
+ if codec == "text":
+ data = obj.get_data()
+ out.write(f'<data size="{len(data)}">{escape(data)}</data>\n')
+ out.write("</stream>")
+ return
+
+ if isinstance(obj, PDFObjRef):
+ out.write(f'<ref id="{obj.objid}" />')
+ return
+
+ if isinstance(obj, PSKeyword):
+ # Likely bug: obj.name is bytes, not str
+ out.write(f"<keyword>{obj.name}</keyword>") # type: ignore [str-bytes-safe]
+ return
+
+ if isinstance(obj, PSLiteral):
+ # Likely bug: obj.name may be bytes, not str
+ out.write(f"<literal>{obj.name}</literal>") # type: ignore [str-bytes-safe]
+ return
+
+ if isnumber(obj):
+ out.write(f"<number>{obj}</number>")
+ return
+
+ raise PDFTypeError(obj)
+
+
+def dumptrailers(
+ out: TextIO,
+ doc: PDFDocument,
+ show_fallback_xref: bool = False,
+) -> None:
+ for xref in doc.xrefs:
+ if not isinstance(xref, PDFXRefFallback) or show_fallback_xref:
+ out.write("<trailer>\n")
+ dumpxml(out, xref.get_trailer())
+ out.write("\n</trailer>\n\n")
+ no_xrefs = all(isinstance(xref, PDFXRefFallback) for xref in doc.xrefs)
+ if no_xrefs and not show_fallback_xref:
+ msg = (
+ "This PDF does not have an xref. Use --show-fallback-xref if "
+ "you want to display the content of a fallback xref that "
+ "contains all objects."
+ )
+ logger.warning(msg)
+
+
+def dumpallobjs(
+ out: TextIO,
+ doc: PDFDocument,
+ codec: str | None = None,
+ show_fallback_xref: bool = False,
+) -> None:
+ visited = set()
+ out.write("<pdf>")
+ for xref in doc.xrefs:
+ for objid in xref.get_objids():
+ if objid in visited:
+ continue
+ visited.add(objid)
+ try:
+ obj = doc.getobj(objid)
+ if obj is None:
+ continue
+ out.write(f'<object id="{objid}">\n')
+ dumpxml(out, obj, codec=codec)
+ out.write("\n</object>\n\n")
+ except PDFObjectNotFound as e:
+ print(f"not found: {e!r}")
+ dumptrailers(out, doc, show_fallback_xref)
+ out.write("</pdf>")
+
+
+def dumpoutline(
+ outfp: TextIO,
+ fname: str,
+ objids: Any,
+ pagenos: Container[int],
+ password: str = "",
+ dumpall: bool = False,
+ codec: str | None = None,
+ extractdir: str | None = None,
+) -> None:
+ with open(fname, "rb") as fp:
+ parser = PDFParser(fp)
+ doc = PDFDocument(parser, password)
+ pages = {
+ page.pageid: pageno
+ for (pageno, page) in enumerate(PDFPage.create_pages(doc), 1)
+ }
+
+ def resolve_dest(dest: object) -> Any:
+ if isinstance(dest, (str, bytes)):
+ dest = resolve1(doc.get_dest(dest))
+ elif isinstance(dest, PSLiteral):
+ dest = resolve1(doc.get_dest(dest.name))
+ if isinstance(dest, dict):
+ dest = dest["D"]
+ if isinstance(dest, PDFObjRef):
+ dest = dest.resolve()
+ return dest
+
+ try:
+ outlines = doc.get_outlines()
+ outfp.write("<outlines>\n")
+ for level, title, dest, a, _se in outlines:
+ pageno = None
+ if dest:
+ dest = resolve_dest(dest)
+ pageno = pages[dest[0].objid]
+ elif a:
+ action = a
+ if isinstance(action, dict):
+ subtype = action.get("S")
+ if subtype and repr(subtype) == "/'GoTo'" and action.get("D"):
+ dest = resolve_dest(action["D"])
+ pageno = pages[dest[0].objid]
+ s = escape(title)
+ outfp.write(f'<outline level="{level!r}" title="{s}">\n')
+ if dest is not None:
+ outfp.write("<dest>")
+ dumpxml(outfp, dest)
+ outfp.write("</dest>\n")
+ if pageno is not None:
+ outfp.write(f"<pageno>{pageno!r}</pageno>\n")
+ outfp.write("</outline>\n")
+ outfp.write("</outlines>\n")
+ except PDFNoOutlines:
+ pass
+ parser.close()
+
+
+LITERAL_FILESPEC = LIT("Filespec")
+LITERAL_EMBEDDEDFILE = LIT("EmbeddedFile")
+
+
+def extractembedded(fname: str, password: str, extractdir: str) -> None:
+ def extract1(objid: int, obj: dict[str, Any]) -> None:
+ filename = os.path.basename(obj.get("UF") or cast(bytes, obj.get("F")).decode())
+ fileref = obj["EF"].get("UF") or obj["EF"].get("F")
+ fileobj = doc.getobj(fileref.objid)
+ if not isinstance(fileobj, PDFStream):
+ error_msg = (
+ f"unable to process PDF: reference for {filename!r} is not a PDFStream"
+ )
+ raise PDFValueError(error_msg)
+ if fileobj.get("Type") is not LITERAL_EMBEDDEDFILE:
+ raise PDFValueError(
+ f"unable to process PDF: reference for {filename!r} "
+ "is not an EmbeddedFile",
+ )
+ path = os.path.join(extractdir, f"{objid:06d}-{filename}")
+ if os.path.exists(path):
+ raise PDFIOError(f"file exists: {path!r}")
+ print(f"extracting: {path!r}")
+ os.makedirs(os.path.dirname(path), exist_ok=True)
+ with open(path, "wb") as out:
+ out.write(fileobj.get_data())
+
+ with open(fname, "rb") as fp:
+ parser = PDFParser(fp)
+ doc = PDFDocument(parser, password)
+ extracted_objids = set()
+ for xref in doc.xrefs:
+ for objid in xref.get_objids():
+ obj = doc.getobj(objid)
+ if (
+ objid not in extracted_objids
+ and isinstance(obj, dict)
+ and obj.get("Type") is LITERAL_FILESPEC
+ ):
+ extracted_objids.add(objid)
+ extract1(objid, obj)
+
+
+def dumppdf(
+ outfp: TextIO,
+ fname: str,
+ objids: Iterable[int],
+ pagenos: Container[int],
+ password: str = "",
+ dumpall: bool = False,
+ codec: str | None = None,
+ extractdir: str | None = None,
+ show_fallback_xref: bool = False,
+) -> None:
+ with open(fname, "rb") as fp:
+ parser = PDFParser(fp)
+ doc = PDFDocument(parser, password)
+ if objids:
+ for objid in objids:
+ obj = doc.getobj(objid)
+ dumpxml(outfp, obj, codec=codec)
+ if pagenos:
+ for pageno, page in enumerate(PDFPage.create_pages(doc)):
+ if pageno in pagenos:
+ if codec:
+ for obj in page.contents:
+ obj = stream_value(obj)
+ dumpxml(outfp, obj, codec=codec)
+ else:
+ dumpxml(outfp, page.attrs)
+ if dumpall:
+ dumpallobjs(outfp, doc, codec, show_fallback_xref)
+ if (not objids) and (not pagenos) and (not dumpall):
+ dumptrailers(outfp, doc, show_fallback_xref)
+ if codec not in ("raw", "binary"):
+ outfp.write("\n")
+
+
+def create_parser() -> ArgumentParser:
+ parser = ArgumentParser(description=__doc__, add_help=True)
+ parser.add_argument(
+ "files",
+ type=str,
+ default=None,
+ nargs="+",
+ help="One or more paths to PDF files.",
+ )
+
+ parser.add_argument(
+ "--version",
+ "-v",
+ action="version",
+ version=f"pdfminer.six v{pdfminer.__version__}",
+ )
+ parser.add_argument(
+ "--debug",
+ "-d",
+ default=False,
+ action="store_true",
+ help="Use debug logging level.",
+ )
+ procedure_parser = parser.add_mutually_exclusive_group()
+ procedure_parser.add_argument(
+ "--extract-toc",
+ "-T",
+ default=False,
+ action="store_true",
+ help="Extract structure of outline",
+ )
+ procedure_parser.add_argument(
+ "--extract-embedded",
+ "-E",
+ type=str,
+ help="Extract embedded files",
+ )
+
+ parse_params = parser.add_argument_group(
+ "Parser",
+ description="Used during PDF parsing",
+ )
+ parse_params.add_argument(
+ "--page-numbers",
+ type=int,
+ default=None,
+ nargs="+",
+ help="A space-seperated list of page numbers to parse.",
+ )
+ parse_params.add_argument(
+ "--pagenos",
+ "-p",
+ type=str,
+ help="A comma-separated list of page numbers to parse. Included for "
+ "legacy applications, use --page-numbers for more idiomatic "
+ "argument entry.",
+ )
+ parse_params.add_argument(
+ "--objects",
+ "-i",
+ type=str,
+ help="Comma separated list of object numbers to extract",
+ )
+ parse_params.add_argument(
+ "--all",
+ "-a",
+ default=False,
+ action="store_true",
+ help="If the structure of all objects should be extracted",
+ )
+ parse_params.add_argument(
+ "--show-fallback-xref",
+ action="store_true",
+ help="Additionally show the fallback xref. Use this if the PDF "
+ "has zero or only invalid xref's. This setting is ignored if "
+ "--extract-toc or --extract-embedded is used.",
+ )
+ parse_params.add_argument(
+ "--password",
+ "-P",
+ type=str,
+ default="",
+ help="The password to use for decrypting PDF file.",
+ )
+
+ output_params = parser.add_argument_group(
+ "Output",
+ description="Used during output generation.",
+ )
+ output_params.add_argument(
+ "--outfile",
+ "-o",
+ type=str,
+ default="-",
+ help='Path to file where output is written. Or "-" (default) to '
+ "write to stdout.",
+ )
+ codec_parser = output_params.add_mutually_exclusive_group()
+ codec_parser.add_argument(
+ "--raw-stream",
+ "-r",
+ default=False,
+ action="store_true",
+ help="Write stream objects without encoding",
+ )
+ codec_parser.add_argument(
+ "--binary-stream",
+ "-b",
+ default=False,
+ action="store_true",
+ help="Write stream objects with binary encoding",
+ )
+ codec_parser.add_argument(
+ "--text-stream",
+ "-t",
+ default=False,
+ action="store_true",
+ help="Write stream objects as plain text",
+ )
+
+ return parser
+
+
+def main(argv: list[str] | None = None) -> None:
+ parser = create_parser()
+ args = parser.parse_args(args=argv)
+
+ if args.debug:
+ logging.getLogger().setLevel(logging.DEBUG)
+
+ objids = [int(x) for x in args.objects.split(",")] if args.objects else []
+
+ if args.page_numbers:
+ pagenos = {x - 1 for x in args.page_numbers}
+ elif args.pagenos:
+ pagenos = {int(x) - 1 for x in args.pagenos.split(",")}
+ else:
+ pagenos = set()
+
+ password = args.password
+
+ if args.raw_stream:
+ codec: str | None = "raw"
+ elif args.binary_stream:
+ codec = "binary"
+ elif args.text_stream:
+ codec = "text"
+ else:
+ codec = None
+
+ # Use context manager for file output, ensuring proper cleanup
+ with sys.stdout if args.outfile == "-" else open(args.outfile, "w") as outfp:
+ for fname in args.files:
+ if args.extract_toc:
+ dumpoutline(
+ outfp,
+ fname,
+ objids,
+ pagenos,
+ password=password,
+ dumpall=args.all,
+ codec=codec,
+ extractdir=None,
+ )
+ elif args.extract_embedded:
+ extractembedded(
+ fname, password=password, extractdir=args.extract_embedded
+ )
+ else:
+ dumppdf(
+ outfp,
+ fname,
+ objids,
+ pagenos,
+ password=password,
+ dumpall=args.all,
+ codec=codec,
+ extractdir=None,
+ show_fallback_xref=args.show_fallback_xref,
+ )
+
+
+if __name__ == "__main__":
+ main()
diff --git a/pip-bins/odf2mht b/pip-bins/odf2mht
new file mode 100755
index 0000000..e10e63b
--- /dev/null
+++ b/pip-bins/odf2mht
@@ -0,0 +1,72 @@
+#!/usr/bin/python3
+# -*- coding: utf-8 -*-
+# Copyright (C) 2006 Søren Roug, European Environment Agency
+#
+# This is free software. You may redistribute it under the terms
+# of the Apache license and the GNU General Public License Version
+# 2 or at your option any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public
+# License along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+#
+# Contributor(s):
+#
+from __future__ import print_function
+from odf.odf2xhtml import ODF2XHTML
+import zipfile
+import sys
+#from time import gmtime, strftime
+
+from email.mime.multipart import MIMEMultipart
+from email.mime.nonmultipart import MIMENonMultipart
+from email.mime.text import MIMEText
+from email import encoders
+
+if sys.version_info[0]==3: unicode=str
+
+if len(sys.argv) != 2:
+ sys.stderr.write("Usage: %s inputfile\n" % sys.argv[0])
+ sys.exit(1)
+
+suffices = {
+ 'wmf':('image','x-wmf'),
+ 'png':('image','png'),
+ 'gif':('image','gif'),
+ 'jpg':('image','jpeg'),
+ 'jpeg':('image','jpeg')
+ }
+
+msg = MIMEMultipart('related',type="text/html")
+# msg['Subject'] = 'Subject here'
+# msg['From'] = '<Saved by ODT2MHT>'
+# msg['Date'] = strftime("%a, %d %b %Y %H:%M:%S +0000", gmtime())
+msg.preamble = 'This is a multi-part message in MIME format.'
+msg.epilogue = ''
+odhandler = ODF2XHTML()
+result = odhandler.odf2xhtml(unicode(sys.argv[1]))
+htmlpart = MIMEText(result,'html','us-ascii')
+htmlpart['Content-Location'] = 'index.html'
+msg.attach(htmlpart)
+z = zipfile.ZipFile(sys.argv[1])
+for file in z.namelist():
+ if file[0:9] == 'Pictures/':
+ suffix = file[file.rfind(".")+1:]
+ main,sub = suffices.get(suffix,('application','octet-stream'))
+ img = MIMENonMultipart(main,sub)
+ img.set_payload(z.read(file))
+ img['Content-Location'] = "" + file
+ encoders.encode_base64(img)
+ msg.attach(img)
+z.close()
+print (msg.as_string())
+
+
+# Local Variables: ***
+# mode: python ***
+# End: ***
diff --git a/pip-bins/odf2xhtml b/pip-bins/odf2xhtml
new file mode 100755
index 0000000..d70dfb0
--- /dev/null
+++ b/pip-bins/odf2xhtml
@@ -0,0 +1,59 @@
+#!/usr/bin/python3
+# -*- coding: utf-8 -*-
+# Copyright (C) 2007 Søren Roug, European Environment Agency
+#
+# This is free software. You may redistribute it under the terms
+# of the Apache license and the GNU General Public License Version
+# 2 or at your option any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public
+# License along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+#
+# Contributor(s):
+#
+from odf.odf2xhtml import ODF2XHTML
+import sys, getopt
+
+if sys.version_info[0]==3: unicode=str
+
+from io import StringIO
+
+def usage():
+ sys.stderr.write("Usage: %s [-p] inputfile\n" % sys.argv[0])
+
+try:
+ opts, args = getopt.getopt(sys.argv[1:], "ep", ["plain","embedable"])
+except getopt.GetoptError:
+ usage()
+ sys.exit(2)
+
+generatecss = True
+embedable = False
+for o, a in opts:
+ if o in ("-p", "--plain"):
+ generatecss = False
+ if o in ("-e", "--embedable"):
+ embedable = True
+
+if len(args) != 1:
+ usage()
+ sys.exit(2)
+
+odhandler = ODF2XHTML(generatecss, embedable)
+try:
+ result = odhandler.odf2xhtml(unicode(args[0]))
+except:
+ sys.stderr.write("Unable to open file %s or file is not OpenDocument\n" % args[0])
+ sys.exit(1)
+sys.stdout.write(result)
+
+
+# Local Variables: ***
+# mode: python ***
+# End: ***
diff --git a/pip-bins/odf2xml b/pip-bins/odf2xml
new file mode 100755
index 0000000..f04d9f1
--- /dev/null
+++ b/pip-bins/odf2xml
@@ -0,0 +1,81 @@
+#!/usr/bin/python3
+# -*- coding: utf-8 -*-
+# Copyright (C) 2008 Søren Roug, European Environment Agency
+#
+# This is free software. You may redistribute it under the terms
+# of the Apache license and the GNU General Public License Version
+# 2 or at your option any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public
+# License along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+#
+# Contributor(s):
+#
+#
+
+# OpenDocument can be a complete office document in a single
+# XML document. This script will create such a document.
+import sys, getopt, base64
+from odf.opendocument import load
+from odf.draw import Image, ObjectOle
+from odf.style import BackgroundImage
+from odf.text import ListLevelStyleImage
+from odf.office import BinaryData
+
+if sys.version_info[0]==3: unicode=str
+
+def usage():
+ sys.stderr.write("Usage: %s [-e] [-o outputfile] [inputfile]\n" % sys.argv[0])
+
+
+if __name__ == "__main__":
+ embedimage = False
+ try:
+ opts, args = getopt.getopt(sys.argv[1:], "o:e", ["output="])
+ except getopt.GetoptError:
+ usage()
+ sys.exit(2)
+
+ outputfile = '-'
+
+ for o, a in opts:
+ if o in ("-o", "--output"):
+ outputfile = a
+ if o == '-e':
+ embedimage = True
+
+ if len(args) > 1:
+ usage()
+ sys.exit(2)
+ if len(args) == 0:
+ d = load(sys.stdin)
+ else:
+ d = load(unicode(args[0]))
+ if embedimage:
+ images = d.getElementsByType(Image) + \
+ d.getElementsByType(BackgroundImage) + \
+ d.getElementsByType(ObjectOle) + \
+ d.getElementsByType(ListLevelStyleImage)
+ for image in images:
+ href = image.getAttribute('href')
+ if href and href[:9] == "Pictures/":
+ p = d.Pictures[href]
+ bp = base64.encodestring(p[1])
+ image.addElement(BinaryData(text=bp))
+ image.removeAttribute('href')
+ xml = d.xml()
+ if outputfile == '-':
+ print (xml)
+ else:
+ open(outputfile,"wb").write(xml)
+
+
+# Local Variables: ***
+# mode: python ***
+# End: ***
diff --git a/pip-bins/odfimgimport b/pip-bins/odfimgimport
new file mode 100755
index 0000000..bd95fed
--- /dev/null
+++ b/pip-bins/odfimgimport
@@ -0,0 +1,190 @@
+#!/usr/bin/python3
+# -*- coding: utf-8 -*-
+# Copyright (C) 2007-2009 Søren Roug, European Environment Agency
+#
+# This is free software. You may redistribute it under the terms
+# of the Apache license and the GNU General Public License Version
+# 2 or at your option any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public
+# License along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+#
+# Contributor(s):
+#
+from __future__ import print_function
+
+import zipfile, sys, getopt, mimetypes
+try:
+ from urllib2 import urlopen, quote, unquote
+except ImportError:
+ from urllib.request import urlopen, quote, unquote
+try:
+ from urlparse import urlunsplit, urlsplit
+except ImportError:
+ from urllib.parse import urlunsplit, urlsplit
+from odf.opendocument import load
+from odf.draw import Image
+
+if sys.version_info[0]==3: unicode=str
+
+#sys.tracebacklimit = 0
+
+# Variable to count the number of retrieval failures
+failures = 0
+
+# Set to one if quiet behaviour is wanted
+quiet = 0
+
+# If set will write every url to import
+verbose = 0
+
+# Dictionary with new pictures. Key is original file path
+# Item is newfilename
+newpictures = {}
+doc = None
+
+def importpicture(href):
+ """ Add the picture to the ZIP file
+ Returns the new path name to the file in the zip archive
+ If it is unable to import, then it returns the original href
+ Sideeffect: add line to manifest
+ """
+ global doc, newpictures, failures, verbose
+
+ # Check that it is not already in the manifest
+ if href in doc.Pictures: return href
+
+ image = None
+ if verbose: print ("Importing", href, file=sys.stderr)
+ if href[:7] == "http://" or href[:8] == "https://" or href[:6] == "ftp://":
+ # There is a bug in urlopen: It can't open urls with non-ascii unicode
+ # characters. Convert to UTF-8 and then use percent encoding
+ try:
+ goodhref = href.encode('ascii')
+ except:
+ o = list(urlsplit(href))
+ o[2] = quote(o[2].encode('utf-8'))
+ goodhref = urlunsplit(o)
+ if goodhref in newpictures:
+ if verbose: print ("already imported", file=sys.stderr)
+ return newpictures[goodhref] # Already imported
+ try:
+ f = urlopen(goodhref.decode("utf-8"))
+ image = f.read()
+ headers = f.info()
+ f.close()
+ # Get the mimetype from the headerlines
+ c_t = headers['Content-Type'].split(';')[0].strip()
+ if c_t: mediatype = c_t.split(';')[0].strip()
+ if verbose: print ("OK", file=sys.stderr)
+ except:
+ failures += 1
+ if verbose: print ("failed", file=sys.stderr)
+ return href
+ # Remove query string
+ try: href= href[:href.rindex('?')]
+ except: pass
+ try:
+ lastslash = href[href.rindex('/'):]
+ ext = lastslash[lastslash.rindex('.'):]
+ except: ext = mimetypes.guess_extension(mediatype)
+ # Everything is a simple path.
+ else:
+ goodhref = href
+ if href[:3] == '../':
+ if directory is None:
+ goodhref = unquote(href[3:])
+ else:
+ goodhref = unquote(directory + href[2:])
+ if goodhref in newpictures:
+ if verbose: print ("already imported", file=sys.stderr)
+ return newpictures[goodhref] # Already imported
+ mediatype, encoding = mimetypes.guess_type(goodhref)
+ if mediatype is None:
+ mediatype = ''
+ try: ext = goodhref[goodhref.rindex('.'):]
+ except: ext=''
+ else:
+ ext = mimetypes.guess_extension(mediatype)
+ try:
+ image = file(goodhref).read()
+ if verbose: print ("OK", file=sys.stderr)
+ except:
+ failures += 1
+ if verbose: print ("failed", file=sys.stderr)
+ return href
+ # If we have a picture to import, the image variable contains it
+ # and manifestfn, ext and mediatype has a value
+ if image:
+ manifestfn = doc.addPictureFromString(image, unicode(mediatype))
+ newpictures[goodhref] = manifestfn
+ return manifestfn
+
+ if verbose: print ("not imported", file=sys.stderr)
+ return href
+
+def exitwithusage(exitcode=2):
+ """ Print out usage information and exit """
+ print ("Usage: %s [-q] [-v] [-o output] [inputfile]" % sys.argv[0], file=sys.stderr)
+ print ("\tInputfile must be OpenDocument format", file=sys.stderr)
+ sys.exit(exitcode)
+
+outputfile = None
+writefile = True
+
+try:
+ opts, args = getopt.getopt(sys.argv[1:], "qvo:")
+except getopt.GetoptError:
+ exitwithusage()
+
+for o, a in opts:
+ if o == "-o":
+ outputfile = a
+ writefile = True
+ if o == "-q":
+ quiet = 1
+ if o == "-v":
+ verbose = 1
+
+if len(args) == 0:
+ try:
+ doc = load(sys.stdin)
+ directory = None
+ except:
+ print ("Couldn't open OpenDocument file", file=sys.stderr)
+ exitwithusage()
+else:
+ fn = unicode(args[0])
+ if not zipfile.is_zipfile(fn):
+ exitwithusage()
+ dirinx = max(fn.rfind('\\'), fn.rfind('/'))
+ if dirinx >= 0: directory = fn[:dirinx]
+ else: directory = "."
+ doc = load(fn)
+
+for image in doc.getElementsByType(Image):
+ href = image.getAttribute('href')
+ newhref = importpicture(href)
+ image.setAttribute('href',newhref)
+
+if writefile:
+ if outputfile is None:
+ doc.save(fn)
+ else:
+ doc.save(unicode(outputfile))
+
+
+if quiet == 0 and failures > 0:
+ print ("Couldn't import %d image(s)" % failures, file=sys.stderr)
+sys.exit( int(failures > 0) )
+
+
+# Local Variables: ***
+# mode: python ***
+# End: ***
diff --git a/pip-bins/odflint b/pip-bins/odflint
new file mode 100755
index 0000000..e5da6bb
--- /dev/null
+++ b/pip-bins/odflint
@@ -0,0 +1,216 @@
+#!/usr/bin/python3
+# -*- coding: utf-8 -*-
+# Copyright (C) 2009 Søren Roug, European Environment Agency
+#
+# This is free software. You may redistribute it under the terms
+# of the Apache license and the GNU General Public License Version
+# 2 or at your option any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public
+# License along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+#
+# Contributor(s):
+#
+import zipfile
+from xml.sax import make_parser,handler
+from xml.sax.xmlreader import InputSource
+import xml.sax.saxutils
+import sys
+from odf.opendocument import OpenDocument
+from odf import element, grammar
+from odf.namespaces import *
+from odf.attrconverters import attrconverters, cnv_string
+
+from io import BytesIO
+
+if sys.version_info[0]==3: unicode=str
+
+extension_attributes = {
+ "OpenOffice.org" : {
+ (METANS,u'template'): (
+ (XLINKNS,u'role'),
+ ),
+ (STYLENS,u'graphic-properties'): (
+ (STYLENS,u'background-transparency'),
+ ),
+ (STYLENS,u'paragraph-properties'): (
+ (TEXTNS,u'enable-numbering'),
+ (STYLENS,u'join-border'),
+ ),
+ (STYLENS,u'table-cell-properties'): (
+ (STYLENS,u'writing-mode'),
+ ),
+ (STYLENS,u'table-row-properties'): (
+ (STYLENS,u'keep-together'),
+ ),
+ },
+ "KOffice" : {
+ (STYLENS,u'graphic-properties'): (
+ (KOFFICENS,u'frame-behavior-on-new-page'),
+ ),
+ (DRAWNS,u'page'): (
+ (KOFFICENS,u'name'),
+ ),
+ (PRESENTATIONNS,u'show-shape'): (
+ (KOFFICENS,u'order-id'),
+ ),
+ (PRESENTATIONNS,u'hide-shape'): (
+ (KOFFICENS,u'order-id'),
+ ),
+ (CHARTNS,u'legend'): (
+ (KOFFICENS,u'title'),
+ ),
+ }
+}
+
+printed_errors = []
+
+def print_error(str):
+ if str not in printed_errors:
+ printed_errors.append(str)
+ print (str)
+
+def chop_arg(arg):
+ if len(arg) > 20:
+ return "%s..." % arg[0:20]
+ return arg
+
+def make_qname(tag):
+ return "%s:%s" % (nsdict.get(tag[0],tag[0]), tag[1])
+
+def allowed_attributes(tag):
+ return grammar.allowed_attributes.get(tag)
+
+
+class ODFElementHandler(handler.ContentHandler):
+ """ Extract headings from content.xml of an ODT file """
+ def __init__(self, document):
+ self.doc = document
+ self.tagstack = []
+ self.data = []
+ self.currtag = None
+
+ def characters(self, data):
+ self.data.append(data)
+
+ def startElementNS(self, tag, qname, attrs):
+ """ Pseudo-create an element
+ """
+ allowed_attrs = grammar.allowed_attributes.get(tag)
+ attrdict = {}
+ for (att,value) in attrs.items():
+ prefix = nsdict.get(att[0],att[0])
+ # Check if it is a known extension
+ notan_extension = True
+ for product, ext_attrs in extension_attributes.items():
+ allowed_ext_attrs = ext_attrs.get(tag)
+ if allowed_ext_attrs and att in allowed_ext_attrs:
+ print_error("Warning: Attribute %s in element <%s> is illegal - %s extension" % ( make_qname(att), make_qname(tag), product))
+ notan_extension = False
+ # Check if it is an allowed attribute
+ if notan_extension and allowed_attrs and att not in allowed_attrs:
+ print_error("Error: Attribute %s:%s is not allowed in element <%s>" % ( prefix, att[1], make_qname(tag)))
+ # Check the value
+ try:
+ convert = attrconverters.get(att, cnv_string)
+ convert(att, value, tag)
+ except ValueError as res:
+ print_error("Error: Bad value '%s' for attribute %s:%s in tag: <%s> - %s" %
+ (chop_arg(value), prefix, att[1], make_qname(tag), res))
+
+ self.tagstack.append(tag)
+ self.data = []
+ # Check that the parent allows this child element
+ if tag not in ( (OFFICENS, 'document'), (OFFICENS, 'document-content'), (OFFICENS, 'document-styles'),
+ (OFFICENS, 'document-meta'), (OFFICENS, 'document-settings'),
+ (MANIFESTNS,'manifest')):
+ try:
+ parent = self.tagstack[-2]
+ allowed_children = grammar.allowed_children.get(parent)
+ except:
+ print_error("Error: This document starts with the wrong tag: <%s>" % make_qname(tag))
+ allowed_children = None
+ if allowed_children and tag not in allowed_children:
+ print_error("Error: Element %s is not allowed in element %s" % ( make_qname(tag), make_qname(parent)))
+ # Test that all mandatory attributes have been added.
+ required = grammar.required_attributes.get(tag)
+ if required:
+ for r in required:
+ if attrs.get(r) is None:
+ print_error("Error: Required attribute missing: %s in <%s>" % (make_qname(r), make_qname(tag)))
+
+
+ def endElementNS(self, tag, qname):
+ self.currtag = self.tagstack.pop()
+ str = ''.join(self.data).strip()
+ # Check that only elements that can take text have text
+ # But only elements we know exist in grammar
+ if tag in grammar.allowed_children:
+ if str != '' and tag not in grammar.allows_text:
+ print_error("Error: %s does not allow text data" % make_qname(tag))
+ self.data = []
+
+class ODFDTDHandler(handler.DTDHandler):
+ def notationDecl(self, name, public_id, system_id):
+ """ Ignore DTDs """
+ print_error("Warning: ODF doesn't use DOCTYPEs")
+
+def exitwithusage(exitcode=2):
+ """ print out usage information """
+ sys.stderr.write("Usage: %s inputfile\n" % sys.argv[0])
+ sys.stderr.write("\tInputfile must be OpenDocument format\n")
+ sys.exit(exitcode)
+
+def lint(odffile):
+ if not zipfile.is_zipfile(odffile):
+ print_error("Error: This is not a zipped file")
+ return
+ zfd = zipfile.ZipFile(odffile)
+ try:
+ mimetype = zfd.read('mimetype')
+ except:
+ mimetype=''
+ d = OpenDocument(unicode(mimetype))
+ first = True
+ for zi in zfd.infolist():
+ if first:
+ if zi.filename == 'mimetype':
+ if zi.compress_type != zipfile.ZIP_STORED:
+ print_error("Error: The 'mimetype' member must be stored - not deflated")
+ if zi.comment != "":
+ print_error("Error: The 'mimetype' member must not have extra header info")
+ else:
+ print_error("Warning: The first member in the archive should be the mimetype")
+ first = False
+ if zi.filename in ('META-INF/manifest.xml', 'content.xml', 'meta.xml', 'styles.xml', 'settings.xml'):
+ content = zfd.read(zi.filename)
+ parser = make_parser()
+ parser.setFeature(handler.feature_namespaces, True)
+ parser.setFeature(handler.feature_external_ges, False)
+ parser.setContentHandler(ODFElementHandler(d))
+ dtdh = ODFDTDHandler()
+ parser.setDTDHandler(dtdh)
+ parser.setErrorHandler(handler.ErrorHandler())
+
+ inpsrc = InputSource()
+ if not isinstance(content, str):
+ content=content
+ inpsrc.setByteStream(BytesIO(content))
+ parser.parse(inpsrc)
+
+
+if len(sys.argv) != 2:
+ exitwithusage()
+lint(unicode(sys.argv[1]))
+
+
+
+# Local Variables: ***
+# mode: python ***
+# End: ***
diff --git a/pip-bins/odfmeta b/pip-bins/odfmeta
new file mode 100755
index 0000000..b99be94
--- /dev/null
+++ b/pip-bins/odfmeta
@@ -0,0 +1,266 @@
+#!/usr/bin/python3
+# -*- coding: utf-8 -*-
+# Copyright (C) 2006-2009 Søren Roug, European Environment Agency
+#
+# This is free software. You may redistribute it under the terms
+# of the Apache license and the GNU General Public License Version
+# 2 or at your option any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public
+# License along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+#
+# Contributor(s):
+#
+import zipfile, time, sys, getopt, re
+import xml.sax, xml.sax.saxutils
+from odf.namespaces import TOOLSVERSION, OFFICENS, XLINKNS, DCNS, METANS
+from io import BytesIO
+
+OUTENCODING="utf-8"
+
+whitespace = re.compile(r'\s+')
+
+fields = {
+'title': (DCNS,u'title'),
+'description': (DCNS,u'description'),
+'subject': (DCNS,u'subject'),
+'creator': (DCNS,u'creator'),
+'date': (DCNS,u'date'),
+'language': (DCNS,u'language'),
+'generator': (METANS,u'generator'),
+'initial-creator': (METANS,u'initial-creator'),
+'keyword': (METANS,u'keyword'),
+'editing-duration': (METANS,u'editing-duration'),
+'editing-cycles': (METANS,u'editing-cycles'),
+'printed-by': (METANS,u'printed-by'),
+'print-date': (METANS,u'print-date'),
+'creation-date': (METANS,u'creation-date'),
+'user-defined': (METANS,u'user-defined'),
+#'template': (METANS,u'template'),
+}
+
+xfields = []
+Xfields = []
+addfields = {}
+deletefields = {}
+yieldfields = {}
+showversion = None
+
+def exitwithusage(exitcode=2):
+ """ print out usage information """
+ sys.stderr.write("Usage: %s [-cdlvV] [-xXaAI metafield]... [-o output] [inputfile]\n" % sys.argv[0])
+ sys.stderr.write("\tInputfile must be OpenDocument format\n")
+ sys.exit(exitcode)
+
+def normalize(str):
+ """
+ The normalize-space function returns the argument string with whitespace
+ normalized by stripping leading and trailing whitespace and replacing
+ sequences of whitespace characters by a single space.
+ """
+ return whitespace.sub(' ', str).strip()
+
+class MetaCollector:
+ """
+ The MetaCollector is a pseudo file object, that can temporarily ignore write-calls
+ It could probably be replaced with a StringIO object.
+ """
+ def __init__(self):
+ self._content = []
+ self.dowrite = True
+
+ def write(self, str):
+ if self.dowrite:
+ self._content.append(str)
+
+ def content(self):
+ return ''.join(self._content)
+
+
+base = xml.sax.saxutils.XMLGenerator
+
+class odfmetaparser(base):
+ """ Parse a meta.xml file with an event-driven parser and replace elements.
+ It would probably be a cleaner approach to use a DOM based parser and
+ then manipulate in memory.
+ Small issue: Reorders elements
+ """
+ version = 'Unknown'
+
+ def __init__(self):
+ self._mimetype = ''
+ self.output = MetaCollector()
+ self._data = []
+ self.seenfields = {}
+ base.__init__(self, self.output, OUTENCODING)
+
+ def startElementNS(self, name, qname, attrs):
+ self._data = []
+ field = name
+# I can't modify the template until the tool replaces elements at the same
+# location and not at the end
+# if name == (METANS,u'template'):
+# self._data = [attrs.get((XLINKNS,u'title'),'')]
+ if showversion and name == (OFFICENS,u'document-meta'):
+ if showversion == '-V':
+ print ("version:%s" % attrs.get((OFFICENS,u'version'),'Unknown').decode('utf-8'))
+ else:
+ print ("%s" % attrs.get((OFFICENS,u'version'),'Unknown').decode('utf-8'))
+ if name == (METANS,u'user-defined'):
+ field = attrs.get((METANS,u'name'))
+ if field in deletefields:
+ self.output.dowrite = False
+ elif field in yieldfields:
+ del addfields[field]
+ base.startElementNS(self, name, qname, attrs)
+ else:
+ base.startElementNS(self, name, qname, attrs)
+ self._tag = field
+
+ def endElementNS(self, name, qname):
+ field = name
+ if name == (METANS,u'user-defined'):
+ field = self._tag
+ if name == (OFFICENS,u'meta'):
+ for k,v in addfields.items():
+ if len(v) > 0:
+ if type(k) == type(''):
+ base.startElementNS(self,(METANS,u'user-defined'),None,{(METANS,u'name'):k})
+ base.characters(self, v)
+ base.endElementNS(self, (METANS,u'user-defined'),None)
+ else:
+ base.startElementNS(self, k, None, {})
+ base.characters(self, v)
+ base.endElementNS(self, k, None)
+ if name in xfields:
+ print ("%s" % self.data())
+ if name in Xfields:
+ if isinstance(self._tag, tuple):
+ texttag = self._tag[1]
+ else:
+ texttag = self._tag
+ print ("%s:%s" % (texttag, self.data()))
+ if field in deletefields:
+ self.output.dowrite = True
+ else:
+ base.endElementNS(self, name, qname)
+
+ def characters(self, content):
+ base.characters(self, content)
+ self._data.append(content)
+
+ def meta(self):
+ return self.output.content()
+
+ def data(self):
+ if usenormalize:
+ return normalize(''.join(self._data))
+ else:
+ return ''.join(self._data)
+
+now = time.localtime()[:6]
+outputfile = "-"
+writemeta = False # Do we change any meta data?
+usenormalize = False
+
+try:
+ opts, args = getopt.getopt(sys.argv[1:], "cdlvVI:A:a:o:x:X:")
+except getopt.GetoptError:
+ exitwithusage()
+
+if len(opts) == 0:
+ opts = [ ('-l','') ]
+
+for o, a in opts:
+ if o in ('-a','-A','-I'):
+ writemeta = True
+ if a.find(":") >= 0:
+ k,v = a.split(":",1)
+ else:
+ k,v = (a, "")
+ if len(k) == 0:
+ exitwithusage()
+ k = fields.get(k,k)
+ addfields[k] = unicode(v,'utf-8')
+ if o == '-a':
+ yieldfields[k] = True
+ if o == '-I':
+ deletefields[k] = True
+ if o == '-d':
+ writemeta = True
+ addfields[(DCNS,u'date')] = "%04d-%02d-%02dT%02d:%02d:%02d" % now
+ deletefields[(DCNS,u'date')] = True
+ if o == '-c':
+ usenormalize = True
+ if o in ('-v', '-V'):
+ showversion = o
+ if o == '-l':
+ Xfields = fields.values()
+ if o == "-x":
+ xfields.append(fields.get(a,a))
+ if o == "-X":
+ Xfields.append(fields.get(a,a))
+ if o == "-o":
+ outputfile = a
+
+# The specification says we should change the element to our own,
+# and must not export the original identifier.
+if writemeta:
+ addfields[(METANS,u'generator')] = TOOLSVERSION
+ deletefields[(METANS,u'generator')] = True
+
+odfs = odfmetaparser()
+parser = xml.sax.make_parser()
+parser.setFeature(xml.sax.handler.feature_namespaces, 1)
+parser.setContentHandler(odfs)
+
+if len(args) == 0:
+ zin = zipfile.ZipFile(sys.stdin,'r')
+else:
+ if not zipfile.is_zipfile(args[0]):
+ exitwithusage()
+ zin = zipfile.ZipFile(args[0], 'r')
+
+try:
+ content = zin.read('meta.xml').decode('utf-8')
+except:
+ sys.stderr.write("File has no meta data\n")
+ sys.exit(1)
+parser.parse(BytesIO(content.encode('utf-8')))
+
+if writemeta:
+ if outputfile == '-':
+ if sys.stdout.isatty():
+ sys.stderr.write("Won't write ODF file to terminal\n")
+ sys.exit(1)
+ zout = zipfile.ZipFile(sys.stdout,"w")
+ else:
+ zout = zipfile.ZipFile(outputfile,"w")
+
+
+
+ # Loop through the input zipfile and copy the content to the output until we
+ # get to the meta.xml. Then substitute.
+ for zinfo in zin.infolist():
+ if zinfo.filename == "meta.xml":
+ # Write meta
+ zi = zipfile.ZipInfo("meta.xml", now)
+ zi.compress_type = zipfile.ZIP_DEFLATED
+ zout.writestr(zi,odfs.meta() )
+ else:
+ payload = zin.read(zinfo.filename)
+ zout.writestr(zinfo, payload)
+
+ zout.close()
+zin.close()
+
+
+# Local Variables: ***
+# mode: python ***
+# End: ***
diff --git a/pip-bins/odfoutline b/pip-bins/odfoutline
new file mode 100755
index 0000000..0d26071
--- /dev/null
+++ b/pip-bins/odfoutline
@@ -0,0 +1,144 @@
+#!/usr/bin/python3
+# -*- coding: utf-8 -*-
+# Copyright (C) 2006 Søren Roug, European Environment Agency
+#
+# This is free software. You may redistribute it under the terms
+# of the Apache license and the GNU General Public License Version
+# 2 or at your option any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public
+# License along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+#
+# Contributor(s):
+#
+from __future__ import print_function
+import zipfile
+from xml.sax import make_parser,handler
+from xml.sax.xmlreader import InputSource
+import xml.sax.saxutils
+import sys
+from odf.namespaces import TEXTNS, TABLENS, DRAWNS
+
+try:
+ from cStringIO import StringIO
+except ImportError:
+ from io import StringIO
+
+
+def getxmlpart(odffile, xmlfile):
+ """ Get the content out of the ODT file"""
+ z = zipfile.ZipFile(odffile)
+ content = z.read(xmlfile)
+ z.close()
+ return content
+
+
+
+#
+# Extract headings from content.xml
+#
+class ODTHeadingHandler(handler.ContentHandler):
+ """ Extract headings from content.xml of an ODT file """
+ def __init__(self, eater):
+ self.r = eater
+ self.data = []
+ self.level = 0
+
+ def characters(self, data):
+ self.data.append(data)
+
+ def startElementNS(self, tag, qname, attrs):
+ if tag == (TEXTNS, 'h'):
+ self.level = 0
+ for (att,value) in attrs.items():
+ if att == (TEXTNS, 'outline-level'):
+ self.level = int(value)
+ self.data = []
+
+ def endElementNS(self, tag, qname):
+ if tag == (TEXTNS, 'h'):
+ str = ''.join(self.data)
+ self.data = []
+ self.r.append("%d%*s%s" % (self.level, self.level, '', str))
+
+class ODTSheetHandler(handler.ContentHandler):
+ """ Extract sheet names from content.xml of an ODS file """
+ def __init__(self, eater):
+ self.r = eater
+
+ def startElementNS(self, tag, qname, attrs):
+ if tag == (TABLENS, 'table'):
+ sheetname = attrs.get((TABLENS, 'name'))
+ if sheetname:
+ self.r.append(sheetname)
+
+class ODTSlideHandler(handler.ContentHandler):
+ """ Extract headings from content.xml of an ODT file """
+ def __init__(self, eater):
+ self.r = eater
+ self.data = []
+ self.pagenum = 0
+
+ def characters(self, data):
+ self.data.append(data)
+
+ def startElementNS(self, tag, qname, attrs):
+ if tag == (DRAWNS, 'page'):
+ self.pagenum = self.pagenum + 1
+ self.r.append("SLIDE %d: %s" % ( self.pagenum, attrs.get((DRAWNS, 'name'),'')))
+ if tag == (TEXTNS, 'p'):
+ self.data = []
+
+ def endElementNS(self, tag, qname):
+ if tag == (TEXTNS, 'p'):
+ str = ''.join(self.data)
+ self.data = []
+ if len(str) > 0:
+ self.r.append(" " + str)
+
+def odtheadings(odtfile):
+ mimetype = getxmlpart(odtfile,'mimetype')
+ content = getxmlpart(odtfile,'content.xml')
+ lines = []
+ parser = make_parser()
+ parser.setFeature(handler.feature_namespaces, 1)
+ if not isinstance(mimetype, str):
+ mimetype=mimetype.decode("utf-8")
+ if mimetype in ('application/vnd.oasis.opendocument.text',
+ 'application/vnd.oasis.opendocument.text-template'):
+ parser.setContentHandler(ODTHeadingHandler(lines))
+ elif mimetype in ('application/vnd.oasis.opendocument.spreadsheet',
+ 'application/vnd.oasis.opendocument.spreadsheet-template'):
+ parser.setContentHandler(ODTSheetHandler(lines))
+ elif mimetype in ('application/vnd.oasis.opendocument.presentation'
+ 'application/vnd.oasis.opendocument.presentation-template'):
+ parser.setContentHandler(ODTSlideHandler(lines))
+ else:
+ print ("Unsupported fileformat")
+ sys.exit(2)
+ parser.setErrorHandler(handler.ErrorHandler())
+
+ inpsrc = InputSource()
+ if not isinstance(content, str):
+ content=content.decode("utf-8")
+ inpsrc.setByteStream(StringIO(content))
+ parser.parse(inpsrc)
+ return lines
+
+
+if __name__ == "__main__":
+ filler = " "
+ for heading in odtheadings(sys.argv[1]):
+ print (heading)
+
+
+
+# Local Variables: ***
+# mode: python ***
+# End: ***
diff --git a/pip-bins/odfuserfield b/pip-bins/odfuserfield
new file mode 100755
index 0000000..cae1574
--- /dev/null
+++ b/pip-bins/odfuserfield
@@ -0,0 +1,101 @@
+#!/usr/bin/python3
+# -*- coding: utf-8 -*-
+# Copyright (C) 2006-2007 Søren Roug, European Environment Agency
+#
+# This is free software. You may redistribute it under the terms
+# of the Apache license and the GNU General Public License Version
+# 2 or at your option any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public
+# License along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+#
+# Contributor(s): Michael Howitz, gocept gmbh & co. kg
+
+import sys
+import getopt
+
+import odf.userfield
+
+if sys.version_info[0]==3: unicode=str
+
+listfields = False
+Listfields = False
+xfields = []
+Xfields = []
+setfields = {}
+outputfile = None
+inputfile = None
+
+
+def exitwithusage(exitcode=2):
+ """ print out usage information """
+ sys.stderr.write("Usage: %s [-lL] [-xX metafield] [-s metafield:value]... "
+ "[-o output] [inputfile]\n" % sys.argv[0])
+ sys.stderr.write("\tInputfile must be OpenDocument format\n")
+ sys.exit(exitcode)
+
+
+try:
+ opts, args = getopt.getopt(sys.argv[1:], "lLs:o:x:X:")
+except getopt.GetoptError:
+ exitwithusage()
+
+if len(opts) == 0:
+ exitwithusage()
+
+for o, a in opts:
+ if o == '-s':
+ if a.find(":") >= 0:
+ k,v = a.split(":",1)
+ else:
+ k,v = (a, "")
+ if len(k) == 0:
+ exitwithusage()
+ setfields[unicode(k)] = unicode(v)
+ if o == '-l':
+ listfields = True
+ Listfields = False
+ if o == '-L':
+ Listfields = True
+ listfields = False
+ if o == "-x":
+ xfields.append(unicode(a))
+ if o == "-X":
+ Xfields.append(unicode(a))
+ if o == "-o":
+ outputfile = unicode(a)
+
+if len(args) != 0:
+ inputfile = unicode(args[0])
+
+user_fields = odf.userfield.UserFields(inputfile, outputfile)
+
+if xfields:
+ for value in user_fields.list_values(xfields):
+ print (value)
+
+if Listfields or Xfields:
+ if Listfields:
+ Xfields = None
+ for field_name, value_type, value in user_fields.list_fields_and_values(
+ Xfields):
+ print ("%s#%s:%s" % (field_name, value_type, value))
+
+if listfields:
+ for value in user_fields.list_fields():
+ print (value)
+
+if setfields:
+ user_fields.update(setfields)
+
+
+
+# Local Variables: ***
+# mode: python ***
+# End: ***
diff --git a/pip-bins/pdf2txt.py b/pip-bins/pdf2txt.py
new file mode 100755
index 0000000..cda5b88
--- /dev/null
+++ b/pip-bins/pdf2txt.py
@@ -0,0 +1,324 @@
+#!/usr/bin/python3
+"""A command line tool for extracting text and images from PDF and
+output it to plain text, html, xml or tags.
+"""
+
+import argparse
+import logging
+import sys
+from collections.abc import Container, Iterable
+from typing import Any
+
+import pdfminer.high_level
+from pdfminer.layout import LAParams
+from pdfminer.pdfexceptions import PDFValueError
+from pdfminer.utils import AnyIO
+
+logging.basicConfig()
+
+OUTPUT_TYPES = ((".htm", "html"), (".html", "html"), (".xml", "xml"), (".tag", "tag"))
+
+
+def float_or_disabled(x: str) -> float | None:
+ if x.lower().strip() == "disabled":
+ return None
+ try:
+ return float(x)
+ except ValueError as err:
+ raise argparse.ArgumentTypeError(f"invalid float value: {x}") from err
+
+
+def extract_text(
+ files: Iterable[str] = [],
+ outfile: str = "-",
+ laparams: LAParams | None = None,
+ output_type: str = "text",
+ codec: str = "utf-8",
+ strip_control: bool = False,
+ maxpages: int = 0,
+ page_numbers: Container[int] | None = None,
+ password: str = "",
+ scale: float = 1.0,
+ rotation: int = 0,
+ layoutmode: str = "normal",
+ output_dir: str | None = None,
+ debug: bool = False,
+ disable_caching: bool = False,
+ **kwargs: Any,
+) -> None:
+ if not files:
+ raise PDFValueError("Must provide files to work upon!")
+
+ if output_type == "text" and outfile != "-":
+ for override, alttype in OUTPUT_TYPES:
+ if outfile.endswith(override):
+ output_type = alttype
+
+ if outfile == "-":
+ outfp: AnyIO = sys.stdout
+ if sys.stdout.encoding is not None:
+ codec = "utf-8"
+ for fname in files:
+ with open(fname, "rb") as fp:
+ pdfminer.high_level.extract_text_to_fp(fp, **locals())
+ else:
+ # Use context manager for file output, ensuring proper cleanup
+ with open(outfile, "wb") as outfp:
+ for fname in files:
+ with open(fname, "rb") as fp:
+ pdfminer.high_level.extract_text_to_fp(fp, **locals())
+
+
+def create_parser() -> argparse.ArgumentParser:
+ parser = argparse.ArgumentParser(description=__doc__, add_help=True)
+ parser.add_argument(
+ "files",
+ type=str,
+ default=None,
+ nargs="+",
+ help="One or more paths to PDF files.",
+ )
+
+ parser.add_argument(
+ "--version",
+ "-v",
+ action="version",
+ version=f"pdfminer.six v{pdfminer.__version__}",
+ )
+ parser.add_argument(
+ "--debug",
+ "-d",
+ default=False,
+ action="store_true",
+ help="Use debug logging level.",
+ )
+ parser.add_argument(
+ "--disable-caching",
+ "-C",
+ default=False,
+ action="store_true",
+ help="If caching or resources, such as fonts, should be disabled.",
+ )
+
+ parse_params = parser.add_argument_group(
+ "Parser",
+ description="Used during PDF parsing",
+ )
+ parse_params.add_argument(
+ "--page-numbers",
+ type=int,
+ default=None,
+ nargs="+",
+ help="A space-seperated list of page numbers to parse.",
+ )
+ parse_params.add_argument(
+ "--pagenos",
+ "-p",
+ type=str,
+ help="A comma-separated list of page numbers to parse. "
+ "Included for legacy applications, use --page-numbers "
+ "for more idiomatic argument entry.",
+ )
+ parse_params.add_argument(
+ "--maxpages",
+ "-m",
+ type=int,
+ default=0,
+ help="The maximum number of pages to parse.",
+ )
+ parse_params.add_argument(
+ "--password",
+ "-P",
+ type=str,
+ default="",
+ help="The password to use for decrypting PDF file.",
+ )
+ parse_params.add_argument(
+ "--rotation",
+ "-R",
+ default=0,
+ type=int,
+ help="The number of degrees to rotate the PDF "
+ "before other types of processing.",
+ )
+
+ la_params = LAParams() # will be used for defaults
+ la_param_group = parser.add_argument_group(
+ "Layout analysis",
+ description="Used during layout analysis.",
+ )
+ la_param_group.add_argument(
+ "--no-laparams",
+ "-n",
+ default=False,
+ action="store_true",
+ help="If layout analysis parameters should be ignored.",
+ )
+ la_param_group.add_argument(
+ "--detect-vertical",
+ "-V",
+ default=la_params.detect_vertical,
+ action="store_true",
+ help="If vertical text should be considered during layout analysis",
+ )
+ la_param_group.add_argument(
+ "--line-overlap",
+ type=float,
+ default=la_params.line_overlap,
+ help="If two characters have more overlap than this they "
+ "are considered to be on the same line. The overlap is specified "
+ "relative to the minimum height of both characters.",
+ )
+ la_param_group.add_argument(
+ "--char-margin",
+ "-M",
+ type=float,
+ default=la_params.char_margin,
+ help="If two characters are closer together than this margin they "
+ "are considered to be part of the same line. The margin is "
+ "specified relative to the width of the character.",
+ )
+ la_param_group.add_argument(
+ "--word-margin",
+ "-W",
+ type=float,
+ default=la_params.word_margin,
+ help="If two characters on the same line are further apart than this "
+ "margin then they are considered to be two separate words, and "
+ "an intermediate space will be added for readability. The margin "
+ "is specified relative to the width of the character.",
+ )
+ la_param_group.add_argument(
+ "--line-margin",
+ "-L",
+ type=float,
+ default=la_params.line_margin,
+ help="If two lines are close together they are considered to "
+ "be part of the same paragraph. The margin is specified "
+ "relative to the height of a line.",
+ )
+ la_param_group.add_argument(
+ "--boxes-flow",
+ "-F",
+ type=float_or_disabled,
+ default=la_params.boxes_flow,
+ help="Specifies how much a horizontal and vertical position of a "
+ "text matters when determining the order of lines. The value "
+ "should be within the range of -1.0 (only horizontal position "
+ "matters) to +1.0 (only vertical position matters). You can also "
+ "pass `disabled` to disable advanced layout analysis, and "
+ "instead return text based on the position of the bottom left "
+ "corner of the text box.",
+ )
+ la_param_group.add_argument(
+ "--all-texts",
+ "-A",
+ default=la_params.all_texts,
+ action="store_true",
+ help="If layout analysis should be performed on text in figures.",
+ )
+
+ output_params = parser.add_argument_group(
+ "Output",
+ description="Used during output generation.",
+ )
+ output_params.add_argument(
+ "--outfile",
+ "-o",
+ type=str,
+ default="-",
+ help="Path to file where output is written. "
+ 'Or "-" (default) to write to stdout.',
+ )
+ output_params.add_argument(
+ "--output_type",
+ "-t",
+ type=str,
+ default="text",
+ help="Type of output to generate {text,html,xml,tag}.",
+ )
+ output_params.add_argument(
+ "--codec",
+ "-c",
+ type=str,
+ default="utf-8",
+ help="Text encoding to use in output file.",
+ )
+ output_params.add_argument(
+ "--output-dir",
+ "-O",
+ default=None,
+ help="The output directory to put extracted images in. If not given, "
+ "images are not extracted.",
+ )
+ output_params.add_argument(
+ "--layoutmode",
+ "-Y",
+ default="normal",
+ type=str,
+ help="Type of layout to use when generating html "
+ "{normal,exact,loose}. If normal,each line is"
+ " positioned separately in the html. If exact"
+ ", each character is positioned separately in"
+ " the html. If loose, same result as normal "
+ "but with an additional newline after each "
+ "text line. Only used when output_type is html.",
+ )
+ output_params.add_argument(
+ "--scale",
+ "-s",
+ type=float,
+ default=1.0,
+ help="The amount of zoom to use when generating html file. "
+ "Only used when output_type is html.",
+ )
+ output_params.add_argument(
+ "--strip-control",
+ "-S",
+ default=False,
+ action="store_true",
+ help="Remove control statement from text. Only used when output_type is xml.",
+ )
+
+ return parser
+
+
+def parse_args(args: list[str] | None) -> argparse.Namespace:
+ parsed_args = create_parser().parse_args(args=args)
+
+ # Propagate parsed layout parameters to LAParams object
+ if parsed_args.no_laparams:
+ parsed_args.laparams = None
+ else:
+ parsed_args.laparams = LAParams(
+ line_overlap=parsed_args.line_overlap,
+ char_margin=parsed_args.char_margin,
+ line_margin=parsed_args.line_margin,
+ word_margin=parsed_args.word_margin,
+ boxes_flow=parsed_args.boxes_flow,
+ detect_vertical=parsed_args.detect_vertical,
+ all_texts=parsed_args.all_texts,
+ )
+
+ if parsed_args.page_numbers:
+ parsed_args.page_numbers = {x - 1 for x in parsed_args.page_numbers}
+
+ if parsed_args.pagenos:
+ parsed_args.page_numbers = {int(x) - 1 for x in parsed_args.pagenos.split(",")}
+
+ if parsed_args.output_type == "text" and parsed_args.outfile != "-":
+ for override, alttype in OUTPUT_TYPES:
+ if parsed_args.outfile.endswith(override):
+ parsed_args.output_type = alttype
+
+ return parsed_args
+
+
+def main(args: list[str] | None = None) -> int:
+ parsed_args = parse_args(args)
+ extract_text(**vars(parsed_args))
+ return 0
+
+
+if __name__ == "__main__":
+ sys.exit(main())
diff --git a/pip-bins/pdfplumber b/pip-bins/pdfplumber
new file mode 100755
index 0000000..cfb94ab
--- /dev/null
+++ b/pip-bins/pdfplumber
@@ -0,0 +1,8 @@
+#!/usr/bin/python3
+# -*- coding: utf-8 -*-
+import re
+import sys
+from pdfplumber.cli import main
+if __name__ == '__main__':
+ sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0])
+ sys.exit(main())
diff --git a/pip-bins/pypdfium2 b/pip-bins/pypdfium2
new file mode 100755
index 0000000..ec81a5f
--- /dev/null
+++ b/pip-bins/pypdfium2
@@ -0,0 +1,8 @@
+#!/usr/bin/python3
+# -*- coding: utf-8 -*-
+import re
+import sys
+from pypdfium2.__main__ import cli_main
+if __name__ == '__main__':
+ sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0])
+ sys.exit(cli_main())
diff --git a/pip-bins/xml2odf b/pip-bins/xml2odf
new file mode 100755
index 0000000..e7114a8
--- /dev/null
+++ b/pip-bins/xml2odf
@@ -0,0 +1,241 @@
+#!/usr/bin/python3
+# -*- coding: utf-8 -*-
+# Copyright (C) 2006 Søren Roug, European Environment Agency
+#
+# This is free software. You may redistribute it under the terms
+# of the Apache license and the GNU General Public License Version
+# 2 or at your option any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public
+# License along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+#
+# Contributor(s):
+#
+#
+
+# OpenDocument can be a complete office document in a single
+# XML document. This script will take such a document and create
+# a package
+import io
+import zipfile,time, sys, getopt
+import xml.sax, xml.sax.saxutils
+from odf import manifest
+
+class SplitWriter:
+ def __init__(self):
+ self.activefiles = []
+ self._content = []
+ self._meta = []
+ self._styles = []
+ self._settings = []
+
+ self.files = {'content': self._content, 'meta': self._meta,
+ 'styles':self._styles, 'settings': self._settings }
+
+ def write(self, str):
+ for f in self.activefiles:
+ f.append(str)
+
+ def activate(self, filename):
+ file = self.files[filename]
+ if file not in self.activefiles:
+ self.activefiles.append(file)
+
+ def deactivate(self, filename):
+ file = self.files[filename]
+ if file in self.activefiles:
+ self.activefiles.remove(file)
+
+odmimetypes = {
+ 'application/vnd.oasis.opendocument.text': '.odt',
+ 'application/vnd.oasis.opendocument.text-template': '.ott',
+ 'application/vnd.oasis.opendocument.graphics': '.odg',
+ 'application/vnd.oasis.opendocument.graphics-template': '.otg',
+ 'application/vnd.oasis.opendocument.presentation': '.odp',
+ 'application/vnd.oasis.opendocument.presentation-template': '.otp',
+ 'application/vnd.oasis.opendocument.spreadsheet': '.ods',
+ 'application/vnd.oasis.opendocument.spreadsheet-template': '.ots',
+ 'application/vnd.oasis.opendocument.chart': '.odc',
+ 'application/vnd.oasis.opendocument.chart-template': '.otc',
+ 'application/vnd.oasis.opendocument.image': '.odi',
+ 'application/vnd.oasis.opendocument.image-template': '.oti',
+ 'application/vnd.oasis.opendocument.formula': '.odf',
+ 'application/vnd.oasis.opendocument.formula-template': '.otf',
+ 'application/vnd.oasis.opendocument.text-master': '.odm',
+ 'application/vnd.oasis.opendocument.text-web': '.oth',
+}
+
+OFFICENS = u"urn:oasis:names:tc:opendocument:xmlns:office:1.0"
+base = xml.sax.saxutils.XMLGenerator
+
+class odfsplitter(base):
+
+ def __init__(self):
+ self._mimetype = ''
+ self.output = SplitWriter()
+ self._prefixes = []
+ base.__init__(self, self.output, 'utf-8')
+
+ def startPrefixMapping(self, prefix, uri):
+ base.startPrefixMapping(self, prefix, uri)
+ self._prefixes.append('xmlns:%s="%s"' % (prefix, uri))
+
+ def startElementNS(self, name, qname, attrs):
+ if name == (OFFICENS, u"document"):
+ self._mimetype = attrs.get((OFFICENS, "mimetype"))
+ elif name == (OFFICENS, u"meta"):
+ self.output.activate('meta')
+
+ elif name == (OFFICENS, u"settings"):
+ self.output.activate('settings')
+ elif name == (OFFICENS, u"scripts"):
+ self.output.activate('content')
+ elif name == (OFFICENS, u"font-face-decls"):
+ self.output.activate('content')
+ self.output.activate('styles')
+ elif name == (OFFICENS, u"styles"):
+ self.output.activate('styles')
+ elif name == (OFFICENS, u"automatic-styles"):
+ self.output.activate('content')
+ self.output.activate('styles')
+ elif name == (OFFICENS, u"master-styles"):
+ self.output.activate('styles')
+ elif name == (OFFICENS, u"body"):
+ self.output.activate('content')
+ base.startElementNS(self, name, qname, attrs)
+
+ def endElementNS(self, name, qname):
+ base.endElementNS(self, name, qname)
+ if name == (OFFICENS, u"meta"):
+ self.output.deactivate('meta')
+ elif name == (OFFICENS, u"settings"):
+ self.output.deactivate('settings')
+ elif name == (OFFICENS, u"scripts"):
+ self.output.deactivate('content')
+ elif name == (OFFICENS, u"font-face-decls"):
+ self.output.deactivate('content')
+ self.output.deactivate('styles')
+ elif name == (OFFICENS, u"styles"):
+ self.output.deactivate('styles')
+ elif name == (OFFICENS, u"automatic-styles"):
+ self.output.deactivate('content')
+ self.output.deactivate('styles')
+ elif name == (OFFICENS, u"master-styles"):
+ self.output.deactivate('styles')
+ elif name == (OFFICENS, u"body"):
+ self.output.deactivate('content')
+
+
+ def content(self):
+ """ Return the content inside a wrapper called <office:document-content>
+ """
+ prefixes = ' '.join(self._prefixes)
+ return ''.join(['<?xml version="1.0" encoding="UTF-8"?>\n<office:document-content %s office:version="1.0">' % prefixes] + list(map(lambda x: x.decode("utf-8"), self.output._content)) + ['</office:document-content>'])
+
+ def settings(self):
+ prefixes = ' '.join(self._prefixes).encode('utf-8')
+ return ''.join( ['<?xml version="1.0" encoding="UTF-8"?>\n<office:document-settings %s office:version="1.0">' % prefixes] + self.output._settings + ['''</office:document-settings>'''])
+
+ def styles(self):
+ prefixes = ' '.join(self._prefixes)
+ return ''.join( ['<?xml version="1.0" encoding="UTF-8"?>\n<office:document-styles %s office:version="1.0">' % prefixes] + list(map(lambda x: x.decode("utf-8"), self.output._styles)) + ['''</office:document-styles>'''])
+
+ def meta(self):
+ prefixes = ' '.join(self._prefixes)
+ return ''.join( ['<?xml version="1.0" encoding="UTF-8"?>\n<office:document-meta %s office:version="1.0">' % prefixes] + list(map(lambda x: x.decode("utf-8"), self.output._meta)) + ['''</office:document-meta>'''])
+
+def usage():
+ sys.stderr.write("Usage: %s [-o outputfile] [-s] inputfile\n" % sys.argv[0])
+
+def manifestxml(m):
+ """ Generates the content of the manifest.xml file """
+ xml=io.StringIO()
+ xml.write(u"<?xml version='1.0' encoding='UTF-8'?>\n")
+ m.toXml(0,xml)
+ return xml.getvalue()
+
+try:
+ opts, args = getopt.getopt(sys.argv[1:], "o:s", ["output=","suffix"])
+except getopt.GetoptError:
+ usage()
+ sys.exit(2)
+
+outputfile = '-'
+addsuffix = False
+
+for o, a in opts:
+ if o in ("-o", "--output"):
+ outputfile = a
+ if o in ("-s", "--suffix"):
+ addsuffix = True
+
+if len(args) > 1:
+ usage()
+ sys.exit(2)
+
+odfs = odfsplitter()
+parser = xml.sax.make_parser()
+parser.setFeature(xml.sax.handler.feature_namespaces, 1)
+parser.setContentHandler(odfs)
+if len(args) == 0:
+ parser.parse(sys.stdin)
+else:
+ parser.parse(open(args[0],"r"))
+
+mimetype = odfs._mimetype
+suffix = odmimetypes.get(mimetype,'.xxx')
+
+if outputfile == '-':
+ if sys.stdout.isatty():
+ sys.stderr.write("Won't write ODF file to terminal\n")
+ sys.exit(1)
+ z = zipfile.ZipFile(sys.stdout,"w")
+else:
+ if addsuffix:
+ outputfile = outputfile + suffix
+ z = zipfile.ZipFile(outputfile,"w")
+
+now = time.localtime()[:6]
+
+# Write mimetype
+zi = zipfile.ZipInfo('mimetype', now)
+zi.compress_type = zipfile.ZIP_STORED
+z.writestr(zi,mimetype)
+
+# Write content
+zi = zipfile.ZipInfo("content.xml", now)
+zi.compress_type = zipfile.ZIP_DEFLATED
+z.writestr(zi,odfs.content() )
+# Write styles
+zi = zipfile.ZipInfo("styles.xml", now)
+zi.compress_type = zipfile.ZIP_DEFLATED
+z.writestr(zi,odfs.styles() )
+
+# Write meta
+zi = zipfile.ZipInfo("meta.xml", now)
+zi.compress_type = zipfile.ZIP_DEFLATED
+z.writestr(zi,odfs.meta() )
+
+m = manifest.Manifest()
+m.addElement(manifest.FileEntry(fullpath="/", mediatype=mimetype))
+m.addElement(manifest.FileEntry(fullpath="content.xml",mediatype="text/xml"))
+m.addElement(manifest.FileEntry(fullpath="styles.xml", mediatype="text/xml"))
+m.addElement(manifest.FileEntry(fullpath="meta.xml", mediatype="text/xml"))
+
+# Write manifest
+zi = zipfile.ZipInfo("META-INF/manifest.xml", now)
+zi.compress_type = zipfile.ZIP_DEFLATED
+z.writestr(zi, manifestxml(m).encode("utf-8") )
+z.close()
+
+
+
+# Local Variables: ***
+# mode: python ***
+# End: ***
diff --git a/simplex-chat b/simplex-chat
new file mode 100755
index 0000000..c697643
--- /dev/null
+++ b/simplex-chat
Binary files differ
diff --git a/sorter/readme.md b/sorter/readme.md
new file mode 100644
index 0000000..e81c411
--- /dev/null
+++ b/sorter/readme.md
@@ -0,0 +1,210 @@
+# Downloads Sorter
+
+A Python script that automatically sorts files from your `~/Downloads` folder into organised subdirectories based on filename keywords and — more importantly — the **actual text content** of each file.
+
+Built for Debian Linux, Polish and English documents, Python 3.11+.
+
+---
+
+## How it works
+
+When you run the script it does the following:
+
+1. Scans every supported file in `~/Downloads` (not subdirectories)
+2. Extracts the full text content from each file
+3. Checks that text against the keyword rules in `rules.toml`
+4. Falls back to checking the filename if no content match is found
+5. Shows you a **dry-run preview** of where each file would go
+6. Asks for confirmation before moving anything
+
+Content always beats filename — so a file called `scan001.pdf` that contains an Arc of Asia invoice will correctly go to `Work/ARC`, even though the filename gives no hint.
+
+### Supported file types
+
+`.pdf` `.docx` `.doc` `.txt` `.xlsx` `.xls` `.md` `.odt`
+
+You can add more in `rules.toml` (see below).
+
+---
+
+## Files
+
+```
+sorter/
+├── sort_downloads.sh # Run this — installs deps, then calls the Python script
+├── sort_downloads.py # The actual logic
+└── rules.toml # Your rules — edit this to add keywords and categories
+```
+
+Keep all three files in the same directory.
+
+---
+
+## Daily use
+
+```bash
+cd ~/.local/bin/sorter
+./sort_downloads.sh
+```
+
+That's it. The script will show a preview, then ask:
+
+```
+Proceed with moving files? [y/N]
+```
+
+Type `y` to move, or just press Enter to cancel without touching anything.
+
+### Useful flags
+
+```bash
+# Skip the confirmation prompt and move immediately
+./sort_downloads.sh --yes
+
+# Preview rules without scanning any files
+./sort_downloads.sh --list-rules
+
+# Sort a different folder instead of ~/Downloads
+./sort_downloads.sh --dir /path/to/folder
+
+# Use a different rules file
+./sort_downloads.sh --config /path/to/other-rules.toml
+```
+
+### Output folders
+
+Subfolders are created automatically inside `~/Downloads` the first time a file routes there. Based on the default rules you will get:
+
+```
+~/Downloads/
+├── Work/
+│ ├── ARC/
+│ └── LKIT/
+├── AKW/
+└── Other/
+```
+
+If two files would land on the same destination path, the script appends `_1`, `_2` etc. rather than overwriting.
+
+---
+
+## How to extend it
+
+Everything is controlled by `rules.toml`. You never need to touch the Python script to add new keywords or categories.
+
+### Adding a keyword to an existing category
+
+Open `rules.toml` and add a line to the relevant list:
+
+```toml
+[[categories]]
+name = "Work/ARC"
+content_keywords = [
+ "arc of asia",
+ "9571181577",
+ "your new keyword here", # ← add here
+]
+filename_keywords = [
+ "arc",
+ "new_filename_hint", # ← or here
+]
+```
+
+`content_keywords` are matched against the full extracted text of the file.
+`filename_keywords` are matched against the filename only, and only used if no content match was found first.
+
+Both are case-insensitive and partial — `"kasprzak"` will match `"Łukasz Kasprzak International Trade"`.
+
+### Adding a new category
+
+Append a new `[[categories]]` block anywhere in the list:
+
+```toml
+[[categories]]
+name = "Finance/Banking"
+description = "Bank statements and account exports"
+content_keywords = [
+ "revolut",
+ "account statement",
+ "wyciąg bankowy",
+ "mbank",
+]
+filename_keywords = [
+ "revolut",
+ "statement",
+ "wyciag",
+]
+```
+
+The folder path (`Finance/Banking`, `Personal/Tax`, or any depth you like) will be created automatically inside `~/Downloads`.
+
+**Order matters** — categories are checked top to bottom and the first match wins. Put more specific categories above broader ones.
+
+### Adding a new file extension
+
+Add it to `supported_extensions` in `rules.toml`:
+
+```toml
+supported_extensions = [
+ ".pdf",
+ ".docx",
+ ".txt",
+ # ... existing entries ...
+ ".log", # plain text — works out of the box
+ ".csv", # plain text — works out of the box
+]
+```
+
+Plain text formats (`.log`, `.csv`, `.json`, `.xml`, `.ini`, `.conf`) work immediately with no code changes. Binary formats that need a dedicated parser (e.g. `.pptx`, `.ods`) would require a small addition to `sort_downloads.py`.
+
+### Changing the catch-all folder
+
+Files that match no category go here:
+
+```toml
+fallback_folder = "Other"
+```
+
+Change it to anything you like, e.g. `"Unsorted"` or `"Inbox"`.
+
+---
+
+## Keyword tips
+
+- **NIP / REGON / KRS numbers** are the most reliable content keywords — they are unique per company and appear on every invoice and document
+- Put **broad keywords** (like `"invoice"`) lower in the list so they don't accidentally catch documents that should match a more specific category above
+- If a bank statement matches the wrong category because a supplier's address appears as a payee, move that supplier's address out of `content_keywords` and rely on the NIP/REGON instead
+- Polish characters work fine in both content and filename keywords (`ł`, `ó`, `ą`, `ś`, `ź`, etc.)
+- Run `./sort_downloads.sh --list-rules` after editing to confirm your changes loaded correctly
+
+---
+
+## Dependencies
+
+| Library | Purpose | Installed by |
+|---|---|---|
+| `pdfplumber` | Read text from PDF files | `sort_downloads.sh` automatically |
+| `python-docx` | Read text from .docx/.doc files | `sort_downloads.sh` automatically |
+| `openpyxl` | Read text from .xlsx/.xls files | `sort_downloads.sh` automatically |
+| `odfpy` | Read text from .odt files | `sort_downloads.sh` automatically |
+| `tomllib` | Parse rules.toml | Built into Python 3.11+ — nothing to install |
+
+The bash wrapper (`sort_downloads.sh`) checks for and installs any missing libraries automatically on each run using `pip install --break-system-packages`.
+
+**Python 3.11 or newer is required.** On Debian 12+ this is the default.
+
+---
+
+## Troubleshooting
+
+**A file landed in the wrong folder**
+Run `--list-rules` to check what keywords are loaded. The preview also shows the match reason in brackets, e.g. `[content: 'wiercany 60a']` — use this to identify which keyword caused the misroute and either remove it or move a more specific category above it in `rules.toml`.
+
+**A file ended up in Other**
+The script found no matching keyword in the file's content or filename. Open the file, find a unique phrase or number, and add it as a `content_keyword` to the appropriate category.
+
+**PDF content is not being read**
+Check that `pdfplumber` is installed (`pip show pdfplumber`). Some PDFs are image-only scans with no embedded text — these cannot be read without OCR, which is not currently supported.
+
+**TOML syntax error on startup**
+TOML is strict about quoting — all strings must be in double quotes. Numbers like NIP/REGON must also be quoted (`"9571181577"`, not `9571181577`) to be treated as text for matching.
diff --git a/sorter/rules.toml b/sorter/rules.toml
new file mode 100644
index 0000000..0c0a7ac
--- /dev/null
+++ b/sorter/rules.toml
@@ -0,0 +1,124 @@
+# ─────────────────────────────────────────────────────────────────────────────
+# Downloads Sorter — Rules Configuration (TOML)
+#
+# PRIORITY: content_keywords > filename_keywords
+# File contents are scanned first; filename is used as fallback.
+#
+# FOLDER PATHS: relative to your Downloads directory.
+# Use / for subfolders, e.g. "Work/ARC"
+#
+# KEYWORD TIPS:
+# - Case-insensitive ("arc" matches "ARC", "Arc", "arc")
+# - Partial match ("kasprzak" matches "Łukasz Kasprzak International Trade")
+# - Always quote strings — avoids TOML type surprises with numbers
+# - NIP/REGON/KRS numbers are the most reliable content keywords
+# - Polish characters work fine (ł, ó, ą, ś, ź, etc.)
+# - Categories are checked in order — first match wins
+# ─────────────────────────────────────────────────────────────────────────────
+
+# Catch-all folder for files that match no category
+fallback_folder = "Other"
+
+# File types to scan (add or remove extensions as needed)
+supported_extensions = [
+ ".pdf",
+ ".docx",
+ ".doc",
+ ".txt",
+ ".xlsx",
+ ".xls",
+ ".md",
+ ".odt",
+]
+
+# ── Categories ────────────────────────────────────────────────────────────────
+# Each [[categories]] block defines one destination folder.
+# Add as many blocks as you need.
+
+[[categories]]
+name = "Work/ARC"
+description = "Arc of Asia Sp. z o.o. — documents, invoices, correspondence"
+content_keywords = [
+ "arc of asia",
+ "arc of asia spółka z ograniczoną odpowiedzialnością",
+ "arc of asia spolka z ograniczona odpowiedzialnoscia",
+ "aleja grunwaldzka 56",
+ "9571181577", # NIP
+ "540356138", # REGON
+ "0001140839", # KRS
+ "pl957118157700000", # EORI
+]
+filename_keywords = [
+ "arc",
+ "aoa",
+ "arc_of_asia",
+]
+
+[[categories]]
+name = "Work/LKIT"
+description = "Łukasz Kasprzak International Trade — documents, invoices, correspondence"
+content_keywords = [
+ "łukasz kasprzak international trade",
+ "lukasz kasprzak international trade",
+ "8181739189", # NIP
+ "540804571", # REGON
+]
+filename_keywords = [
+ "lkit",
+ "kasprzak",
+]
+
+[[categories]]
+name = "AKW"
+description = "Akademia Katolicka w Warszawie"
+content_keywords = [
+ "akademia katolicka w warszawie",
+ "akademia katolicka",
+]
+filename_keywords = [
+ "akw",
+]
+
+# ── Add more categories below ─────────────────────────────────────────────────
+
+# [[categories]]
+# name = "Finance/Banking"
+# description = "Bank statements, Revolut, account history"
+# content_keywords = [
+# "revolut",
+# "account statement",
+# "wyciąg bankowy",
+# ]
+# filename_keywords = [
+# "revolut",
+# "statement",
+# "wyciag",
+# ]
+
+# [[categories]]
+# name = "Finance/Invoices"
+# description = "VAT invoices and receipts"
+# content_keywords = [
+# "faktura vat",
+# "invoice",
+# "23% vat",
+# ]
+# filename_keywords = [
+# "faktura",
+# "invoice",
+# ]
+
+# [[categories]]
+# name = "Personal/Tax"
+# description = "Tax documents and PIT forms"
+# content_keywords = [
+# "pit-37",
+# "pit-36",
+# "urząd skarbowy",
+# "zeznanie podatkowe",
+# ]
+# filename_keywords = [
+# "pit",
+# "podatek",
+# "tax",
+# ]
diff --git a/sorter/sort_downloads.py b/sorter/sort_downloads.py
new file mode 100644
index 0000000..10cc696
--- /dev/null
+++ b/sorter/sort_downloads.py
@@ -0,0 +1,309 @@
+#!/usr/bin/env python3
+"""
+Downloads Folder Sorter
+Rules are loaded from rules.toml — edit that file to add keywords/categories.
+Content match takes priority over filename match.
+Requires Python 3.11+ (uses built-in tomllib).
+"""
+
+import sys
+import shutil
+import tomllib
+import argparse
+from pathlib import Path
+
+# ── Optional deps (graceful fallback) ────────────────────────────────────────
+try:
+ import pdfplumber
+ HAS_PDF = True
+except ImportError:
+ HAS_PDF = False
+
+try:
+ from docx import Document
+ HAS_DOCX = True
+except ImportError:
+ HAS_DOCX = False
+
+try:
+ import openpyxl
+ HAS_XLSX = True
+except ImportError:
+ HAS_XLSX = False
+
+try:
+ from odf import teletype
+ from odf.opendocument import load as odf_load
+ HAS_ODT = True
+except ImportError:
+ HAS_ODT = False
+
+# ── Config loading ────────────────────────────────────────────────────────────
+
+def load_config(config_path: Path) -> dict:
+ if not config_path.exists():
+ print(f"Error: Config file not found: {config_path}")
+ print("Make sure rules.toml is in the same folder as this script.")
+ sys.exit(1)
+ with open(config_path, "rb") as f:
+ config = tomllib.load(f)
+ return config
+
+def validate_config(config: dict):
+ if "categories" not in config or not config["categories"]:
+ print("Error: 'categories' is missing or empty in rules.toml")
+ sys.exit(1)
+ for cat in config["categories"]:
+ if "name" not in cat:
+ print("Error: A category in rules.toml is missing a 'name' field.")
+ sys.exit(1)
+
+# ── Text extraction ───────────────────────────────────────────────────────────
+
+def extract_text_pdf(path: Path) -> str:
+ if not HAS_PDF:
+ return ""
+ try:
+ with pdfplumber.open(path) as pdf:
+ return "\n".join(page.extract_text() or "" for page in pdf.pages)
+ except Exception:
+ return ""
+
+def extract_text_docx(path: Path) -> str:
+ if not HAS_DOCX:
+ return ""
+ try:
+ doc = Document(path)
+ return "\n".join(p.text for p in doc.paragraphs)
+ except Exception:
+ return ""
+
+def extract_text_xlsx(path: Path) -> str:
+ if not HAS_XLSX:
+ return ""
+ try:
+ wb = openpyxl.load_workbook(path, read_only=True, data_only=True)
+ parts = []
+ for ws in wb.worksheets:
+ for row in ws.iter_rows(values_only=True):
+ parts.append(" ".join(str(c) for c in row if c is not None))
+ return "\n".join(parts)
+ except Exception:
+ return ""
+
+def extract_text_odt(path: Path) -> str:
+ if not HAS_ODT:
+ return ""
+ try:
+ doc = odf_load(path)
+ return teletype.extractText(doc.text)
+ except Exception:
+ return ""
+
+def extract_text_plain(path: Path) -> str:
+ for enc in ("utf-8", "utf-16", "latin-1"):
+ try:
+ return path.read_text(encoding=enc)
+ except Exception:
+ continue
+ return ""
+
+def extract_text(path: Path) -> str:
+ ext = path.suffix.lower()
+ if ext == ".pdf":
+ return extract_text_pdf(path)
+ elif ext in (".docx", ".doc"):
+ return extract_text_docx(path)
+ elif ext in (".xlsx", ".xls"):
+ return extract_text_xlsx(path)
+ elif ext == ".odt":
+ return extract_text_odt(path)
+ elif ext in (".txt", ".md"):
+ return extract_text_plain(path)
+ return ""
+
+# ── Classification ────────────────────────────────────────────────────────────
+
+def classify(text: str, filename: str, categories: list, fallback: str) -> tuple:
+ """
+ Returns (destination_folder, match_reason).
+ Content keywords checked first, then filename keywords.
+ Categories are evaluated in order — first match wins.
+ """
+ text_low = text.lower()
+ name_low = filename.lower()
+
+ for cat in categories:
+ for kw in cat.get("content_keywords", []):
+ if str(kw).lower() in text_low:
+ return cat["name"], f"content: '{kw}'"
+
+ for cat in categories:
+ for kw in cat.get("filename_keywords", []):
+ if str(kw).lower() in name_low:
+ return cat["name"], f"filename: '{kw}'"
+
+ return fallback, "no match"
+
+# ── Core ──────────────────────────────────────────────────────────────────────
+
+def collect_files(downloads: Path, extensions: set) -> list:
+ files = []
+ for entry in downloads.iterdir():
+ if entry.is_file() and entry.suffix.lower() in extensions:
+ files.append(entry)
+ return sorted(files)
+
+def plan_moves(files: list, downloads: Path, config: dict) -> list:
+ categories = config["categories"]
+ fallback = config.get("fallback_folder", "Other")
+ moves = []
+ for f in files:
+ print(f" Scanning: {f.name} ...", end=" ", flush=True)
+ text = extract_text(f)
+ dest_folder, reason = classify(text, f.name, categories, fallback)
+ dest_path = downloads / dest_folder / f.name
+ moves.append((f, dest_path, reason))
+ print(f"-> {dest_folder}/ ({reason})")
+ return moves
+
+def print_plan(moves: list, downloads: Path):
+ print("\n" + "=" * 70)
+ print(" DRY-RUN PREVIEW")
+ print("=" * 70)
+ col_w = max(len(src.name) for src, _, _ in moves) + 2
+ for src, dst, reason in moves:
+ rel = dst.relative_to(downloads)
+ print(f" {src.name:<{col_w}} -> {rel} [{reason}]")
+ print("=" * 70)
+
+def execute_moves(moves: list, downloads: Path) -> list:
+ errors = []
+ for src, dst, _ in moves:
+ try:
+ dst.parent.mkdir(parents=True, exist_ok=True)
+ final_dst = dst
+ counter = 1
+ while final_dst.exists():
+ final_dst = dst.with_stem(f"{dst.stem}_{counter}")
+ counter += 1
+ shutil.move(str(src), str(final_dst))
+ print(f" OK {src.name} -> {final_dst.relative_to(downloads)}")
+ except Exception as e:
+ errors.append((src, e))
+ print(f" ERR {src.name} -> {e}")
+ return errors
+
+def check_deps() -> list:
+ missing = []
+ if not HAS_PDF:
+ missing.append("pdfplumber")
+ if not HAS_DOCX:
+ missing.append("python-docx")
+ if not HAS_XLSX:
+ missing.append("openpyxl")
+ if not HAS_ODT:
+ missing.append("odfpy")
+ return missing
+
+# ── Entry point ───────────────────────────────────────────────────────────────
+
+def main():
+ if sys.version_info < (3, 11):
+ print("Error: Python 3.11 or newer is required (for built-in tomllib).")
+ print(f" You have Python {sys.version}")
+ sys.exit(1)
+
+ parser = argparse.ArgumentParser(
+ description="Sort ~/Downloads using rules defined in rules.toml"
+ )
+ parser.add_argument(
+ "--dir",
+ default=str(Path.home() / "Downloads"),
+ help="Path to Downloads folder (default: ~/Downloads)",
+ )
+ parser.add_argument(
+ "--config",
+ default=None,
+ help="Path to rules.toml (default: same folder as this script)",
+ )
+ parser.add_argument(
+ "--yes", "-y",
+ action="store_true",
+ help="Skip confirmation prompt and move immediately",
+ )
+ parser.add_argument(
+ "--list-rules",
+ action="store_true",
+ help="Print loaded categories and keywords, then exit",
+ )
+ args = parser.parse_args()
+
+ script_dir = Path(__file__).parent
+ config_path = Path(args.config) if args.config else script_dir / "rules.toml"
+ config = load_config(config_path)
+ validate_config(config)
+
+ extensions = set(config.get("supported_extensions", [
+ ".pdf", ".docx", ".doc", ".txt", ".xlsx", ".xls", ".md", ".odt"
+ ]))
+
+ if args.list_rules:
+ print(f"\nConfig: {config_path}\n")
+ for cat in config["categories"]:
+ print(f" [folder] {cat['name']}")
+ if cat.get("description"):
+ print(f" {cat['description']}")
+ if cat.get("content_keywords"):
+ kws = ", ".join(str(k) for k in cat["content_keywords"])
+ print(f" content : {kws}")
+ if cat.get("filename_keywords"):
+ kws = ", ".join(str(k) for k in cat["filename_keywords"])
+ print(f" filename: {kws}")
+ print()
+ print(f" [folder] {config.get('fallback_folder', 'Other')} (catch-all)\n")
+ return
+
+ downloads = Path(args.dir).expanduser().resolve()
+ if not downloads.is_dir():
+ print(f"Error: '{downloads}' is not a directory.")
+ sys.exit(1)
+
+ missing = check_deps()
+ if missing:
+ print("Warning: some content-extraction libraries are missing.")
+ print(" Install them for full content scanning:")
+ for m in missing:
+ print(f" pip install {m}")
+ print()
+
+ print(f"Scanning: {downloads}")
+ print(f"Rules: {config_path}\n")
+
+ files = collect_files(downloads, extensions)
+ if not files:
+ print("No supported files found.")
+ sys.exit(0)
+
+ print(f"Found {len(files)} file(s). Classifying...\n")
+ moves = plan_moves(files, downloads, config)
+
+ print_plan(moves, downloads)
+
+ if not args.yes:
+ answer = input("\nProceed with moving files? [y/N] ").strip().lower()
+ if answer != "y":
+ print("Aborted. No files were moved.")
+ sys.exit(0)
+
+ print("\nMoving files...\n")
+ errors = execute_moves(moves, downloads)
+
+ print()
+ if errors:
+ print(f"Done with {len(errors)} error(s).")
+ else:
+ print("All files moved successfully.")
+
+if __name__ == "__main__":
+ main()
diff --git a/sorter/sort_downloads.sh b/sorter/sort_downloads.sh
new file mode 100755
index 0000000..d2023a8
--- /dev/null
+++ b/sorter/sort_downloads.sh
@@ -0,0 +1,60 @@
+#!/usr/bin/env bash
+# sort_downloads.sh — wrapper for sort_downloads.py
+# Installs missing Python deps, then runs the sorter.
+
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+PYTHON_SCRIPT="$SCRIPT_DIR/sort_downloads.py"
+
+# ── Colour helpers ────────────────────────────────────────────────────────────
+GREEN='\033[0;32m'; YELLOW='\033[1;33m'; RED='\033[0;31m'; NC='\033[0m'
+info() { echo -e "${GREEN}[sort]${NC} $*"; }
+warn() { echo -e "${YELLOW}[warn]${NC} $*"; }
+error() { echo -e "${RED}[error]${NC} $*" >&2; }
+
+# ── Check Python 3.11+ ────────────────────────────────────────────────────────
+if ! command -v python3 &>/dev/null; then
+ error "python3 not found. Install it with: sudo apt install python3"
+ exit 1
+fi
+
+PYVER=$(python3 -c "import sys; print(f'{sys.version_info.major}.{sys.version_info.minor}')")
+PYMAJ=$(python3 -c "import sys; print(sys.version_info.major)")
+PYMIN=$(python3 -c "import sys; print(sys.version_info.minor)")
+
+if [[ "$PYMAJ" -lt 3 ]] || [[ "$PYMAJ" -eq 3 && "$PYMIN" -lt 11 ]]; then
+ error "Python 3.11+ required for built-in tomllib. You have Python $PYVER."
+ exit 1
+fi
+
+info "Using Python $(python3 --version) — tomllib built-in, no extra config deps needed."
+
+# ── Install pip if missing ────────────────────────────────────────────────────
+if ! python3 -m pip --version &>/dev/null; then
+ warn "pip not found — installing..."
+ sudo apt-get install -y python3-pip
+fi
+
+# ── Install content-extraction libraries ─────────────────────────────────────
+# Note: pyyaml is no longer needed — TOML is built into Python 3.11+
+declare -A DEPS=(
+ [pdfplumber]="pdfplumber"
+ [docx]="python-docx"
+ [openpyxl]="openpyxl"
+ [odf]="odfpy"
+)
+
+for import_name in "${!DEPS[@]}"; do
+ pip_name="${DEPS[$import_name]}"
+ if ! python3 -c "import $import_name" &>/dev/null 2>&1; then
+ info "Installing $pip_name..."
+ python3 -m pip install --quiet "$pip_name" --break-system-packages
+ fi
+done
+
+info "All dependencies ready."
+echo ""
+
+# ── Run the sorter ────────────────────────────────────────────────────────────
+exec python3 "$PYTHON_SCRIPT" "$@"
diff --git a/straper/README.md b/straper/README.md
new file mode 100644
index 0000000..a076f8f
--- /dev/null
+++ b/straper/README.md
@@ -0,0 +1,123 @@
+# sanctum / labunix rebuild toolkit
+
+A modular, idempotent, Unix-style rebuild toolkit designed to reduce recovery time from weeks to hours.
+
+## Design goals
+
+- Separate **install**, **restore**, and **verify**.
+- Keep base connectivity stable before touching overlay networking.
+- Make secrets and identities opt-in.
+- Never abort the whole run because one restore item failed.
+- Back up before overwrite and write a machine-readable report.
+- Work on both **VM** and **real hardware**.
+
+## Files
+
+- `lib/common.sh` — shared helpers
+- `install-base.sh` — installs packages and base structure only
+- `restore-configs.sh` — restores configs from `capture-full.sh` database
+- `doctor.sh` — read-only health and readiness checks
+
+## Defaults
+
+- Safe by default
+- No automatic identity transplant
+- No blind `/etc/network` overwrite unless explicitly requested
+- No immutable `/etc/resolv.conf`
+- No automatic WireGuard activation unless explicitly requested
+
+## Roles
+
+- `lab` — VM / test clone, least invasive
+- `hardware` — real machine, still conservative
+- `replacement` — real disaster-recovery target, identities allowed when requested
+
+## Recommended recovery order
+
+### 1. Base install
+
+```bash
+sudo ./install-base.sh --role replacement --profile core --start-safe-services
+```
+
+### 2. Restore essentials first
+
+```bash
+sudo ./restore-configs.sh \
+ --role replacement \
+ --category system-basics \
+ --category users \
+ --category ssh \
+ --category apt-sources \
+ --start-services
+```
+
+### 3. Restore network/DNS/firewall carefully
+
+```bash
+sudo ./restore-configs.sh \
+ --role replacement \
+ --category network-base \
+ --category dns \
+ --category firewall \
+ --network-mode source \
+ --dns-mode chain \
+ --start-services
+```
+
+### 4. Restore services
+
+```bash
+sudo ./restore-configs.sh \
+ --role replacement \
+ --category nginx \
+ --category mariadb \
+ --category postfix \
+ --category prosody \
+ --category docker \
+ --restore-secrets \
+ --start-services
+```
+
+### 5. Restore identities last
+
+```bash
+sudo ./restore-configs.sh \
+ --role replacement \
+ --category privacy \
+ --category tls \
+ --category identities \
+ --restore-secrets \
+ --restore-identities \
+ --start-services
+```
+
+### 6. Verify
+
+```bash
+sudo ./doctor.sh --role replacement --strict
+```
+
+## Reports and state
+
+Each run writes:
+
+- `/var/log/labunix-rebuild/report-<RUN_ID>.tsv`
+- `/var/log/labunix-rebuild/doctor-<RUN_ID>.txt`
+- `/var/lib/labunix-rebuild/state-<RUN_ID>.env`
+- `/var/lib/labunix-rebuild/backups/<RUN_ID>/...`
+
+## Notes
+
+- `install-base.sh` does **not** restore your server identity.
+- `restore-configs.sh` is intentionally interactive unless `--yes` is used.
+- `doctor.sh` is read-only.
+- For network restore, use local console access when possible.
+
+
+## Latest patch notes
+
+- v0.1.1 fixes early exit in `load_optional_config()` under `set -e`
+- `install-base.sh` now has a bootstrap DNS fallback for broken `systemd-resolved` stub setups
+- initial `apt-get update` is now validated and fails loudly instead of being reported as success
+- locale handling no longer requires locale tools before base packages install them
diff --git a/straper/common.sh b/straper/common.sh
new file mode 100644
index 0000000..32f3581
--- /dev/null
+++ b/straper/common.sh
@@ -0,0 +1,330 @@
+#!/usr/bin/env bash
+# shellcheck shell=bash
+# Shared helpers for the labunix/sanctum rebuild toolkit.
+
+COMMON_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)"
+TOOLKIT_DIR="$(cd -- "${COMMON_DIR}/.." && pwd)"
+
+TOOLKIT_NAME="labunix-rebuild"
+TOOLKIT_VERSION="0.1.1"
+
+: "${DB_DIR:=/srv/sanctum-rebuild}"
+: "${PUBLIC_DIR:=${DB_DIR}/db/public}"
+: "${SECRET_DIR:=${DB_DIR}/db/secret}"
+: "${ROLE:=lab}" # lab | hardware | replacement
+: "${PROFILE:=core}" # minimal | core | full
+: "${RUN_ID:=$(date +%Y%m%dT%H%M%S)}"
+: "${STATE_DIR:=/var/lib/${TOOLKIT_NAME}}"
+: "${LOG_DIR:=/var/log/${TOOLKIT_NAME}}"
+: "${BACKUP_DIR:=${STATE_DIR}/backups/${RUN_ID}}"
+: "${REPORT_FILE:=${LOG_DIR}/report-${RUN_ID}.tsv}"
+: "${STATE_FILE:=${STATE_DIR}/state-${RUN_ID}.env}"
+: "${DRY_RUN:=false}"
+: "${ASSUME_YES:=false}"
+: "${VERBOSE:=false}"
+: "${START_SERVICES:=false}"
+: "${RESTORE_IDENTITIES:=false}"
+: "${RESTORE_SECRETS:=false}"
+: "${NETWORK_MODE:=safe}" # safe | source
+: "${DNS_MODE:=auto}" # auto | resolved | dnsmasq | unbound | chain
+: "${RESTORE_PURGE:=false}"
+
+umask 022
+
+now_iso() { date +"%Y-%m-%dT%H:%M:%S%z"; }
+log() { printf '[%s] %s\n' "$(now_iso)" "$*"; }
+warn() { printf '[%s] WARN: %s\n' "$(now_iso)" "$*" >&2; }
+die() { printf '[%s] ERROR: %s\n' "$(now_iso)" "$*" >&2; exit 1; }
+
+ensure_root() {
+ [[ ${EUID} -eq 0 ]] || die "must be run as root"
+}
+
+ensure_runtime_dirs() {
+ mkdir -p -- "${STATE_DIR}" "${LOG_DIR}" "${BACKUP_DIR}"
+ touch -- "${REPORT_FILE}" "${STATE_FILE}"
+ chmod 700 -- "${STATE_DIR}" "${BACKUP_DIR}"
+ chmod 755 -- "${LOG_DIR}"
+ if [[ ! -s ${REPORT_FILE} ]]; then
+ printf 'timestamp\tscript\tcategory\titem\taction\tstatus\tnote\tbackup\n' > "${REPORT_FILE}"
+ fi
+}
+
+load_optional_config() {
+ local conf
+ for conf in /etc/labunix/rebuild.conf /root/.config/labunix/rebuild.conf; do
+ if [[ -f ${conf} ]]; then
+ # shellcheck disable=SC1090
+ source "${conf}"
+ fi
+ done
+ return 0
+}
+
+require_cmd() {
+ local c
+ for c in "$@"; do
+ command -v -- "${c}" >/dev/null 2>&1 || die "required command not found: ${c}"
+ done
+}
+
+have_cmd() {
+ command -v -- "$1" >/dev/null 2>&1
+}
+
+set_state() {
+ local key="$1" value="$2" tmp
+ tmp="$(mktemp)"
+ if [[ -f ${STATE_FILE} ]]; then
+ grep -v -E "^${key}=" "${STATE_FILE}" > "${tmp}" || true
+ fi
+ printf '%s=%q\n' "${key}" "${value}" >> "${tmp}"
+ mv -- "${tmp}" "${STATE_FILE}"
+ chmod 600 -- "${STATE_FILE}"
+}
+
+get_state() {
+ local key="$1"
+ [[ -f ${STATE_FILE} ]] || return 1
+ awk -F= -v k="${key}" '$1==k {sub(/^[^=]*=/,""); print; found=1} END {exit(found?0:1)}' "${STATE_FILE}"
+}
+
+report() {
+ local script="$1" category="$2" item="$3" action="$4" status="$5" note="$6" backup="${7:-}"
+ printf '%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n' \
+ "$(now_iso)" "${script}" "${category}" "${item}" "${action}" "${status}" "${note}" "${backup}" \
+ >> "${REPORT_FILE}"
+}
+
+run() {
+ if [[ ${DRY_RUN} == true ]]; then
+ printf '[dry] %s\n' "$*"
+ return 0
+ fi
+ "$@"
+}
+
+run_capture() {
+ local outfile="$1"; shift
+ if [[ ${DRY_RUN} == true ]]; then
+ printf '[dry] capture %s <= %s\n' "${outfile}" "$*"
+ return 0
+ fi
+ "$@" > "${outfile}" 2>&1
+}
+
+backup_target() {
+ local target="$1" rel backup
+ [[ -e ${target} || -L ${target} ]] || return 1
+ rel="${target#/}"
+ backup="${BACKUP_DIR}/${rel}"
+ mkdir -p -- "$(dirname -- "${backup}")"
+ cp -a -- "${target}" "${backup}"
+ printf '%s\n' "${backup}"
+}
+
+files_equal() {
+ local src="$1" dst="$2"
+ [[ -f ${src} && -f ${dst} ]] || return 1
+ cmp -s -- "${src}" "${dst}"
+}
+
+dirs_equal() {
+ local src="$1" dst="$2"
+ [[ -d ${src} && -d ${dst} ]] || return 1
+ diff -qr -- "${src}" "${dst}" >/dev/null 2>&1
+}
+
+copy_path() {
+ local src="$1" dst="$2"
+ if [[ -d ${src} ]]; then
+ mkdir -p -- "${dst}"
+ if [[ ${RESTORE_PURGE} == true ]]; then
+ rsync -a --delete -- "${src}/" "${dst}/"
+ else
+ rsync -a -- "${src}/" "${dst}/"
+ fi
+ else
+ mkdir -p -- "$(dirname -- "${dst}")"
+ cp -a -- "${src}" "${dst}"
+ fi
+}
+
+restore_path() {
+ local script="$1" category="$2" item="$3" src="$4" dst="$5" mode="${6:-}" owner="${7:-}" group="${8:-}"
+ local backup="" changed_note="restored"
+
+ if [[ ! -e ${src} && ! -L ${src} ]]; then
+ report "${script}" "${category}" "${item}" "restore" "skipped" "source missing: ${src}" ""
+ return 0
+ fi
+
+ if [[ -f ${src} && -f ${dst} ]] && files_equal "${src}" "${dst}"; then
+ if [[ -z ${mode} && -z ${owner} && -z ${group} ]]; then
+ report "${script}" "${category}" "${item}" "restore" "ok" "already up to date" ""
+ return 0
+ fi
+ changed_note="metadata normalized"
+ fi
+
+ if [[ -d ${src} && -d ${dst} ]] && dirs_equal "${src}" "${dst}"; then
+ if [[ -z ${mode} && -z ${owner} && -z ${group} ]]; then
+ report "${script}" "${category}" "${item}" "restore" "ok" "already up to date" ""
+ return 0
+ fi
+ changed_note="metadata normalized"
+ fi
+
+ if [[ -e ${dst} || -L ${dst} ]]; then
+ if [[ ${DRY_RUN} == true ]]; then
+ backup="${BACKUP_DIR}/${dst#/}"
+ printf '[dry] backup %s -> %s\n' "${dst}" "${backup}"
+ else
+ backup="$(backup_target "${dst}")"
+ fi
+ fi
+
+ if [[ ${DRY_RUN} == true ]]; then
+ printf '[dry] restore %s -> %s\n' "${src}" "${dst}"
+ [[ -n ${mode} ]] && printf '[dry] chmod %s %s\n' "${mode}" "${dst}"
+ [[ -n ${owner} ]] && printf '[dry] chown -R %s%s %s\n' "${owner}" "${group:+:${group}}" "${dst}"
+ else
+ if [[ "${changed_note}" == "metadata normalized" ]]; then
+ :
+ else
+ copy_path "${src}" "${dst}"
+ fi
+ [[ -n ${mode} ]] && chmod "${mode}" -- "${dst}" 2>/dev/null || true
+ [[ -n ${owner} ]] && chown -R "${owner}${group:+:${group}}" -- "${dst}" 2>/dev/null || true
+ fi
+
+ report "${script}" "${category}" "${item}" "restore" "changed" "${changed_note}" "${backup}"
+ return 0
+}
+
+prompt_yes_no() {
+ local prompt="$1" default="${2:-no}" answer=""
+ if [[ ${ASSUME_YES} == true ]]; then
+ return 0
+ fi
+ case "${default}" in
+ yes) read -r -p "${prompt} [Y/n]: " answer ;;
+ no) read -r -p "${prompt} [y/N]: " answer ;;
+ *) read -r -p "${prompt} [y/n]: " answer ;;
+ esac
+ answer="${answer:-${default}}"
+ [[ ${answer} =~ ^([Yy]|[Yy][Ee][Ss])$ ]]
+}
+
+list_from_file() {
+ local f="$1"
+ [[ -f ${f} ]] || return 0
+ grep -Ev '^(#|$)' "${f}"
+}
+
+apt_install_if_missing() {
+ local pkgs=() pkg
+ for pkg in "$@"; do
+ dpkg -s -- "${pkg}" >/dev/null 2>&1 || pkgs+=("${pkg}")
+ done
+ [[ ${#pkgs[@]} -eq 0 ]] && return 0
+ if [[ ${DRY_RUN} == true ]]; then
+ printf '[dry] apt-get install -y --no-install-recommends %s\n' "${pkgs[*]}"
+ else
+ DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends "${pkgs[@]}"
+ fi
+}
+
+systemctl_safe_enable() {
+ local unit="$1"
+ if ! systemctl list-unit-files --type=service --type=target --type=timer --type=socket | awk '{print $1}' | grep -qx -- "${unit}"; then
+ return 1
+ fi
+ run systemctl enable "${unit}"
+}
+
+systemctl_safe_start() {
+ local unit="$1"
+ if ! systemctl list-unit-files --type=service --type=target --type=timer --type=socket | awk '{print $1}' | grep -qx -- "${unit}"; then
+ return 1
+ fi
+ run systemctl restart "${unit}"
+}
+
+validate_sshd() { have_cmd sshd && sshd -t; }
+validate_nginx() { have_cmd nginx && nginx -t; }
+validate_nft() { [[ -f /etc/nftables.conf ]] && have_cmd nft && nft -c -f /etc/nftables.conf; }
+validate_unbound() { have_cmd unbound-checkconf && unbound-checkconf; }
+validate_dnsmasq() { have_cmd dnsmasq && dnsmasq --test; }
+validate_postfix() { have_cmd postfix && postfix check; }
+validate_mariadb() { have_cmd mariadbd && mariadbd --verbose --help >/dev/null 2>&1; }
+
+service_is_active() {
+ systemctl is-active --quiet "$1"
+}
+
+service_is_enabled() {
+ systemctl is-enabled --quiet "$1"
+}
+
+detect_virtualization() {
+ if have_cmd systemd-detect-virt; then
+ systemd-detect-virt || true
+ else
+ true
+ fi
+}
+
+primary_iface() {
+ ip route get 1.1.1.1 2>/dev/null | awk '/dev/ {for(i=1;i<=NF;i++) if ($i=="dev") {print $(i+1); exit}}'
+}
+
+primary_src_ip() {
+ ip route get 1.1.1.1 2>/dev/null | awk '/src/ {for(i=1;i<=NF;i++) if ($i=="src") {print $(i+1); exit}}'
+}
+
+has_default_route() {
+ ip route show default | grep -q .
+}
+
+can_reach_ip() {
+ local ip="$1"
+ ping -c1 -W3 "${ip}" >/dev/null 2>&1
+}
+
+can_resolve_name() {
+ local name="$1"
+ getent ahostsv4 "${name}" >/dev/null 2>&1
+}
+
+guess_dns_owner() {
+ if service_is_active unbound 2>/dev/null && service_is_active dnsmasq 2>/dev/null; then
+ printf 'chain\n'
+ elif service_is_active unbound 2>/dev/null; then
+ printf 'unbound\n'
+ elif service_is_active dnsmasq 2>/dev/null; then
+ printf 'dnsmasq\n'
+ elif service_is_active systemd-resolved 2>/dev/null; then
+ printf 'resolved\n'
+ else
+ printf 'unknown\n'
+ fi
+}
+
+role_allows_identities() {
+ [[ ${ROLE} == replacement ]]
+}
+
+role_allows_overlay() {
+ [[ ${ROLE} == replacement || ${ROLE} == hardware ]]
+}
+
+mark_manual() {
+ local script="$1" category="$2" item="$3" note="$4"
+ report "${script}" "${category}" "${item}" "manual" "manual" "${note}" ""
+}
+
+note_failure() {
+ local script="$1" category="$2" item="$3" action="$4" note="$5" backup="${6:-}"
+ report "${script}" "${category}" "${item}" "${action}" "failed" "${note}" "${backup}"
+}
diff --git a/straper/doctor.sh b/straper/doctor.sh
new file mode 100644
index 0000000..6aed53e
--- /dev/null
+++ b/straper/doctor.sh
@@ -0,0 +1,222 @@
+#!/usr/bin/env bash
+set -Eeuo pipefail
+
+SCRIPT_DIR="$(cd -- "$(dirname -- "$0")" && pwd)"
+# shellcheck disable=SC1091
+source "${SCRIPT_DIR}/lib/common.sh"
+
+SCRIPT_NAME="doctor.sh"
+STRICT=false
+OUTFILE=""
+
+usage() {
+ cat <<USAGE
+Usage: sudo ./doctor.sh [options]
+
+Purpose:
+ Read-only health and readiness checks for a rebuilt sanctum/labunix host.
+
+Options:
+ --db-dir PATH
+ --role lab|hardware|replacement
+ --strict Exit non-zero if any FAIL is found
+ --output PATH Write human report to this path
+ --dry-run Still read-only; only affects report/state writes
+ --help
+USAGE
+}
+
+while [[ $# -gt 0 ]]; do
+ case "$1" in
+ --db-dir) DB_DIR="$2"; PUBLIC_DIR="${DB_DIR}/db/public"; SECRET_DIR="${DB_DIR}/db/secret"; shift ;;
+ --role) ROLE="$2"; shift ;;
+ --strict) STRICT=true ;;
+ --output) OUTFILE="$2"; shift ;;
+ --dry-run) DRY_RUN=true ;;
+ --help) usage; exit 0 ;;
+ *) die "unknown option: $1" ;;
+ esac
+ shift
+done
+
+ensure_root
+load_optional_config
+ensure_runtime_dirs
+require_cmd ip systemctl ss awk sed grep
+
+OUTFILE="${OUTFILE:-${LOG_DIR}/doctor-${RUN_ID}.txt}"
+: > "${OUTFILE}"
+
+FAILS=0
+WARNS=0
+OKS=0
+MANUALS=0
+
+emit() {
+ printf '%s\n' "$*" | tee -a "${OUTFILE}"
+}
+
+ok_line() {
+ ((OKS+=1))
+ emit "OK | $1"
+ report "${SCRIPT_NAME}" doctor "$1" check ok "$1" ""
+}
+warn_line() {
+ ((WARNS+=1))
+ emit "WARN | $1"
+ report "${SCRIPT_NAME}" doctor "$1" check warn "$1" ""
+}
+fail_line() {
+ ((FAILS+=1))
+ emit "FAIL | $1"
+ report "${SCRIPT_NAME}" doctor "$1" check failed "$1" ""
+}
+manual_line() {
+ ((MANUALS+=1))
+ emit "MANUAL| $1"
+ report "${SCRIPT_NAME}" doctor "$1" manual manual "$1" ""
+}
+
+service_report() {
+ local svc="$1"
+ local unit="$svc"
+ local load active enabled
+
+ [[ "${unit}" == *.* ]] || unit="${unit}.service"
+
+ load="$(systemctl show -P LoadState "${unit}" 2>/dev/null || true)"
+ if [[ -z "${load}" || "${load}" == "not-found" ]]; then
+ warn_line "service ${unit} not installed"
+ return 0
+ fi
+
+ active="$(systemctl show -P ActiveState "${unit}" 2>/dev/null || true)"
+ enabled="$(systemctl show -P UnitFileState "${unit}" 2>/dev/null || true)"
+
+ if [[ "${active}" == "active" ]]; then
+ ok_line "service ${unit} active (${enabled})"
+ else
+ warn_line "service ${unit} ${active:-unknown} (${enabled:-unknown})"
+ fi
+}
+
+emit "labunix/sanctum rebuild doctor"
+emit "timestamp: $(now_iso)"
+emit "role: ${ROLE}"
+emit "db: ${DB_DIR}"
+emit ""
+
+virt="$(detect_virtualization)"
+if [[ -n ${virt} ]]; then
+ warn_line "virtualization detected: ${virt}"
+else
+ ok_line "bare metal or virtualization not detected"
+fi
+
+current_host="$(hostname -s 2>/dev/null || true)"
+captured_host="$(head -n1 "${PUBLIC_DIR}/system/etc-hostname" 2>/dev/null || true)"
+if [[ -n ${captured_host} && ${current_host} == ${captured_host} ]]; then
+ ok_line "hostname matches DB (${current_host})"
+elif [[ -n ${captured_host} ]]; then
+ warn_line "hostname differs: current=${current_host} captured=${captured_host}"
+else
+ warn_line "captured hostname not found in DB"
+fi
+
+if has_default_route; then
+ ok_line "default route present"
+else
+ fail_line "no default route"
+fi
+
+iface="$(primary_iface)"
+src_ip="$(primary_src_ip)"
+[[ -n ${iface} ]] && ok_line "primary route uses iface=${iface} src=${src_ip}" || fail_line "could not determine primary interface via ip route get 1.1.1.1"
+
+if can_reach_ip 1.1.1.1; then
+ ok_line "outbound IP reachability to 1.1.1.1"
+else
+ fail_line "cannot reach 1.1.1.1"
+fi
+
+if can_resolve_name deb.debian.org; then
+ ok_line "DNS resolves deb.debian.org"
+else
+ fail_line "DNS resolution failed for deb.debian.org"
+fi
+
+owner="$(guess_dns_owner)"
+case "${owner}" in
+ resolved) ok_line "DNS owner appears to be systemd-resolved" ;;
+ dnsmasq) ok_line "DNS owner appears to be dnsmasq" ;;
+ unbound) ok_line "DNS owner appears to be unbound" ;;
+ chain) warn_line "DNS owner appears chained (dnsmasq + unbound); verify port ownership intentionally" ;;
+ *) warn_line "DNS owner unclear" ;;
+esac
+
+failed_units="$(systemctl list-units --state=failed --no-legend --plain 2>/dev/null | awk '{print $1}')"
+if [[ -z ${failed_units} ]]; then
+ ok_line "no failed systemd units"
+else
+ fail_line "failed units present: $(echo "${failed_units}" | paste -sd, -)"
+fi
+
+# Config validators
+have_cmd sshd && validate_sshd && ok_line "sshd config validates" || warn_line "sshd config validation failed or sshd missing"
+have_cmd nginx && validate_nginx && ok_line "nginx config validates" || warn_line "nginx config validation failed or nginx missing"
+have_cmd nft && [[ -f /etc/nftables.conf ]] && validate_nft && ok_line "nftables config validates" || warn_line "nftables config validation failed or /etc/nftables.conf missing"
+have_cmd unbound-checkconf && validate_unbound && ok_line "unbound config validates" || warn_line "unbound config validation failed or unbound missing"
+have_cmd dnsmasq && validate_dnsmasq && ok_line "dnsmasq config validates" || warn_line "dnsmasq config validation failed or dnsmasq missing"
+have_cmd postfix && validate_postfix && ok_line "postfix config validates" || warn_line "postfix config validation failed or postfix missing"
+
+emit ""
+emit "Service snapshot"
+service_report ssh
+service_report systemd-resolved
+service_report dnsmasq
+service_report unbound
+service_report nftables
+service_report nginx
+service_report mariadb
+service_report postfix
+service_report prosody
+service_report docker
+service_report prometheus
+service_report grafana-server
+service_report tor
+service_report i2pd
+
+emit ""
+emit "Listening ports snapshot"
+ss -tlnp 2>/dev/null | tee -a "${OUTFILE}" >/dev/null || true
+
+if have_cmd docker; then
+ emit ""
+ emit "Docker snapshot"
+ docker ps --format 'table {{.Names}}\t{{.Status}}\t{{.Ports}}' 2>/dev/null | tee -a "${OUTFILE}" >/dev/null || warn_line "docker present but docker ps failed"
+fi
+
+if [[ -d ${PUBLIC_DIR}/docker/compose-redacted || -d ${SECRET_DIR}/docker/compose-full ]]; then
+ local_count="$(find /srv /opt /home /root -maxdepth 4 \( -name 'docker-compose.yml' -o -name 'docker-compose.yaml' -o -name 'compose.yml' -o -name 'compose.yaml' \) 2>/dev/null | wc -l | tr -d ' ')"
+ ok_line "compose files found locally: ${local_count}"
+fi
+
+if [[ -f /etc/resolv.conf ]]; then
+ emit ""
+ emit "/etc/resolv.conf"
+ sed -n '1,20p' /etc/resolv.conf | tee -a "${OUTFILE}" >/dev/null || true
+fi
+
+if [[ -f ${STATE_DIR}/state-${RUN_ID}.env ]]; then
+ manual_line "current run state file: ${STATE_DIR}/state-${RUN_ID}.env"
+fi
+
+emit ""
+emit "Summary: ok=${OKS} warn=${WARNS} fail=${FAILS} manual=${MANUALS}"
+
+set_state doctor_done true
+report "${SCRIPT_NAME}" run finish exit ok "ok=${OKS}; warn=${WARNS}; fail=${FAILS}; manual=${MANUALS}" ""
+
+if [[ ${STRICT} == true && ${FAILS} -gt 0 ]]; then
+ exit 1
+fi
diff --git a/straper/install-base.sh b/straper/install-base.sh
new file mode 100644
index 0000000..e2a2d42
--- /dev/null
+++ b/straper/install-base.sh
@@ -0,0 +1,342 @@
+#!/usr/bin/env bash
+set -Eeuo pipefail
+
+SCRIPT_DIR="$(cd -- "$(dirname -- "$0")" && pwd)"
+# shellcheck disable=SC1091
+source "${SCRIPT_DIR}/lib/common.sh"
+
+SCRIPT_NAME="install-base.sh"
+USE_DB_APT_SOURCES=false
+FROM_DB_MANUAL=false
+HOSTNAME_WANTED=""
+TIMEZONE_WANTED="Europe/Warsaw"
+LOCALE_WANTED="en_US.UTF-8"
+
+usage() {
+ cat <<USAGE
+Usage: sudo ./install-base.sh [options]
+
+Purpose:
+ Install a reproducible Debian base for sanctum/labunix recovery without
+ importing host-specific configs or identities.
+
+Options:
+ --db-dir PATH Capture DB root (default: ${DB_DIR})
+ --role lab|hardware|replacement
+ --profile minimal|core|full
+ --hostname NAME Set hostname
+ --timezone TZ Set timezone (default: ${TIMEZONE_WANTED})
+ --locale LOCALE Set locale (default: ${LOCALE_WANTED})
+ --use-db-apt-sources Restore apt sources from db/public/packages/apt
+ --from-db-manual Install packages listed in apt-mark-manual.txt
+ --start-safe-services Enable/start low-risk services after install
+ --yes Non-interactive yes to prompts
+ --dry-run Print actions only
+ --verbose More logging
+ --help
+USAGE
+}
+
+while [[ $# -gt 0 ]]; do
+ case "$1" in
+ --db-dir) DB_DIR="$2"; PUBLIC_DIR="${DB_DIR}/db/public"; SECRET_DIR="${DB_DIR}/db/secret"; shift ;;
+ --role) ROLE="$2"; shift ;;
+ --profile) PROFILE="$2"; shift ;;
+ --hostname) HOSTNAME_WANTED="$2"; shift ;;
+ --timezone) TIMEZONE_WANTED="$2"; shift ;;
+ --locale) LOCALE_WANTED="$2"; shift ;;
+ --use-db-apt-sources) USE_DB_APT_SOURCES=true ;;
+ --from-db-manual) FROM_DB_MANUAL=true ;;
+ --start-safe-services) START_SERVICES=true ;;
+ --yes) ASSUME_YES=true ;;
+ --dry-run) DRY_RUN=true ;;
+ --verbose) VERBOSE=true ;;
+ --help) usage; exit 0 ;;
+ *) die "unknown option: $1" ;;
+ esac
+ shift
+done
+
+ensure_root
+load_optional_config
+ensure_runtime_dirs
+require_cmd apt-get apt-cache dpkg hostnamectl rsync
+
+mkdir -p -- /etc/labunix
+
+log "${SCRIPT_NAME}: role=${ROLE} profile=${PROFILE} db=${DB_DIR}"
+report "${SCRIPT_NAME}" "run" "start" "init" "ok" "role=${ROLE} profile=${PROFILE}" ""
+
+case "${ROLE}" in
+ lab|hardware|replacement) ;;
+ *) die "invalid role: ${ROLE}" ;;
+esac
+case "${PROFILE}" in
+ minimal|core|full) ;;
+ *) die "invalid profile: ${PROFILE}" ;;
+esac
+
+BASE_PACKAGES=(
+ ca-certificates curl wget gnupg2 jq rsync vim zsh sudo
+ locales tzdata lsb-release apt-transport-https
+ net-tools iproute2 iputils-ping dnsutils ethtool
+ openssh-server systemd-resolved nftables fail2ban
+)
+
+CORE_PACKAGES=(
+ wireguard wireguard-tools dnsmasq unbound nginx mariadb-server postfix prosody
+ tor i2pd pygopherd mumble-server prometheus prometheus-node-exporter loki
+ apparmor apparmor-utils
+)
+
+FULL_EXTRA_PACKAGES=(
+ git tmux htop build-essential pkg-config
+ docker.io
+)
+
+bootstrap_dns_prepare() {
+ local test_host="${BOOTSTRAP_TEST_HOST:-deb.debian.org}"
+ local backup="/etc/resolv.conf.labunix-preinstall.bak"
+
+ if getent ahostsv4 "${test_host}" >/dev/null 2>&1; then
+ report "${SCRIPT_NAME}" "dns" "bootstrap" "check" "ok" "resolver already working" ""
+ return 0
+ fi
+
+ if ! ping -c 1 -W 3 1.1.1.1 >/dev/null 2>&1; then
+ note_failure "${SCRIPT_NAME}" "dns" "bootstrap" "check" "no outbound IP connectivity"
+ die "bootstrap DNS check failed: no outbound IP connectivity"
+ fi
+
+ if command -v dig >/dev/null 2>&1 && dig +time=2 +tries=1 +short @1.1.1.1 "${test_host}" >/dev/null 2>&1; then
+ [[ -e /etc/resolv.conf && ! -e "${backup}" ]] && cp -a /etc/resolv.conf "${backup}" || true
+ printf 'nameserver 1.1.1.1
+nameserver 9.9.9.9
+' > /etc/resolv.conf
+ chmod 644 /etc/resolv.conf || true
+
+ if getent ahostsv4 "${test_host}" >/dev/null 2>&1; then
+ report "${SCRIPT_NAME}" "dns" "bootstrap" "fallback" "changed" "static resolv.conf applied" "${backup}"
+ return 0
+ fi
+ fi
+
+ note_failure "${SCRIPT_NAME}" "dns" "bootstrap" "check" "DNS still broken after fallback attempt"
+ die "bootstrap DNS check failed"
+}
+
+install_pkg_group() {
+ local group_name="$1"; shift
+ local pkgs=("$@")
+ log "installing package group: ${group_name}"
+ if apt_install_if_missing "${pkgs[@]}"; then
+ report "${SCRIPT_NAME}" "packages" "${group_name}" "install" "ok" "group processed" ""
+ else
+ note_failure "${SCRIPT_NAME}" "packages" "${group_name}" "install" "apt install failed"
+ die "package group failed: ${group_name}"
+ fi
+}
+
+apply_hostname_locale_timezone() {
+ if [[ -n ${HOSTNAME_WANTED} ]]; then
+ if [[ ${DRY_RUN} == true ]]; then
+ printf '[dry] hostnamectl set-hostname %s\n' "${HOSTNAME_WANTED}"
+ else
+ hostnamectl set-hostname "${HOSTNAME_WANTED}"
+ printf '%s\n' "${HOSTNAME_WANTED}" > /etc/hostname
+ fi
+ report "${SCRIPT_NAME}" "system" "hostname" "set" "changed" "${HOSTNAME_WANTED}" ""
+ fi
+
+ if [[ -f /etc/locale.gen ]] && grep -Eq "^#?\s*${LOCALE_WANTED}\b" /etc/locale.gen 2>/dev/null; then
+ if [[ ${DRY_RUN} == true ]]; then
+ printf '[dry] enable locale %s in /etc/locale.gen\n' "${LOCALE_WANTED}"
+ else
+ if have_cmd locale-gen && have_cmd update-locale; then
+ sed -i "s/^# *\(${LOCALE_WANTED}\b.*\)/\1/" /etc/locale.gen
+ locale-gen "${LOCALE_WANTED}" >/dev/null
+ update-locale LANG="${LOCALE_WANTED}" >/dev/null
+ else
+ report "${SCRIPT_NAME}" "system" "locale" "manual" "manual" "locale tools missing; install locales first or rerun after base packages" ""
+ return 0
+ fi
+ fi
+ report "${SCRIPT_NAME}" "system" "locale" "set" "changed" "${LOCALE_WANTED}" ""
+ else
+ report "${SCRIPT_NAME}" "system" "locale" "manual" "manual" "locale missing in /etc/locale.gen: ${LOCALE_WANTED}" ""
+ fi
+
+ if [[ ${DRY_RUN} == true ]]; then
+ printf '[dry] set timezone %s\n' "${TIMEZONE_WANTED}"
+ else
+ ln -sf "/usr/share/zoneinfo/${TIMEZONE_WANTED}" /etc/localtime
+ dpkg-reconfigure -f noninteractive tzdata >/dev/null
+ fi
+ report "${SCRIPT_NAME}" "system" "timezone" "set" "changed" "${TIMEZONE_WANTED}" ""
+}
+
+restore_apt_sources_from_db() {
+ local src="${PUBLIC_DIR}/packages/apt"
+ [[ -d ${src} ]] || { mark_manual "${SCRIPT_NAME}" "apt" "sources" "db/public/packages/apt missing"; return 0; }
+
+ if ! prompt_yes_no "Restore APT sources from DB now?" no; then
+ report "${SCRIPT_NAME}" "apt" "sources" "restore" "skipped" "operator skipped" ""
+ return 0
+ fi
+
+ restore_path "${SCRIPT_NAME}" "apt" "sources.list" "${src}/sources.list" "/etc/apt/sources.list"
+ restore_path "${SCRIPT_NAME}" "apt" "sources.list.d" "${src}/sources.list.d" "/etc/apt/sources.list.d"
+ restore_path "${SCRIPT_NAME}" "apt" "preferences" "${src}/preferences" "/etc/apt/preferences"
+ restore_path "${SCRIPT_NAME}" "apt" "preferences.d" "${src}/preferences.d" "/etc/apt/preferences.d"
+ restore_path "${SCRIPT_NAME}" "apt" "apt.conf.d" "${src}/apt.conf.d" "/etc/apt/apt.conf.d"
+
+ if [[ ${DRY_RUN} == true ]]; then
+ printf '[dry] apt-get update\n'
+ else
+ if apt-get update; then
+ report "${SCRIPT_NAME}" "apt" "update-after-restore" "update" "ok" "apt metadata refreshed" ""
+ else
+ note_failure "${SCRIPT_NAME}" "apt" "update-after-restore" "update" "apt-get update failed after restoring sources"
+ fi
+ fi
+}
+
+package_available() {
+ local pkg="$1"
+ apt-cache show -- "${pkg}" >/dev/null 2>&1
+}
+
+install_from_db_manual() {
+ local f="${PUBLIC_DIR}/packages/apt-mark-manual.txt" pkg installed=0 skipped=0 failed=0
+ [[ -f ${f} ]] || { mark_manual "${SCRIPT_NAME}" "packages" "db-manual" "apt-mark-manual.txt missing"; return 0; }
+
+ while IFS= read -r pkg; do
+ [[ -n ${pkg} ]] || continue
+ [[ ${pkg} =~ ^# ]] && continue
+ if dpkg -s -- "${pkg}" >/dev/null 2>&1; then
+ ((installed+=1))
+ continue
+ fi
+ if ! package_available "${pkg}"; then
+ ((skipped+=1))
+ report "${SCRIPT_NAME}" "packages" "${pkg}" "install" "skipped" "package not available in current apt sources" ""
+ continue
+ fi
+ if [[ ${DRY_RUN} == true ]]; then
+ printf '[dry] apt-get install -y --no-install-recommends %s\n' "${pkg}"
+ ((installed+=1))
+ continue
+ fi
+ if DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends "${pkg}"; then
+ ((installed+=1))
+ report "${SCRIPT_NAME}" "packages" "${pkg}" "install" "changed" "installed from apt-mark-manual" ""
+ else
+ ((failed+=1))
+ note_failure "${SCRIPT_NAME}" "packages" "${pkg}" "install" "apt install failed"
+ fi
+ done < <(list_from_file "${f}")
+
+ report "${SCRIPT_NAME}" "packages" "db-manual-summary" "install" "ok" "installed=${installed} skipped=${skipped} failed=${failed}" ""
+}
+normalize_service_baseline() {
+ local keep_active=(
+ ssh.service
+ systemd-resolved.service
+ fail2ban.service
+ )
+
+ local stop_disable=(
+ dnsmasq.service
+ unbound.service
+ unbound-resolvconf.service
+ nginx.service
+ mariadb.service
+ postfix.service
+ prosody.service
+ prometheus.service
+ tor.service
+ i2pd.service
+ mumble-server.service
+ pygopherd.service
+ )
+
+ local unit
+
+ for unit in "${stop_disable[@]}"; do
+ if systemctl list-unit-files --type=service --no-legend --no-pager 2>/dev/null | awk '{print $1}' | grep -qx -- "${unit}"; then
+ systemctl disable --now "${unit}" >/dev/null 2>&1 || true
+ fi
+ done
+
+ systemctl reset-failed "${stop_disable[@]}" >/dev/null 2>&1 || true
+
+ for unit in "${keep_active[@]}"; do
+ if systemctl list-unit-files --type=service --no-legend --no-pager 2>/dev/null | awk '{print $1}' | grep -qx -- "${unit}"; then
+ systemctl enable "${unit}" >/dev/null 2>&1 || true
+ systemctl restart "${unit}" >/dev/null 2>&1 || true
+ fi
+ done
+
+ report "${SCRIPT_NAME}" "services" "baseline" "normalize" "changed" "safe services kept; app/conflicting services stopped" ""
+}
+start_safe_services() {
+ local svc
+ for svc in ssh systemd-resolved fail2ban; do
+ systemctl_safe_enable "${svc}" || true
+ systemctl_safe_start "${svc}" || true
+ done
+
+ case "${DNS_MODE}" in
+ resolved)
+ systemctl_safe_enable systemd-resolved || true
+ systemctl_safe_start systemd-resolved || true
+ ;;
+ esac
+ report "${SCRIPT_NAME}" "services" "safe-start" "start" "ok" "safe services processed" ""
+}
+
+if [[ ${DRY_RUN} == false ]]; then
+ bootstrap_dns_prepare
+ if apt-get update; then
+ report "${SCRIPT_NAME}" "apt" "update" "update" "ok" "initial apt update" ""
+ else
+ note_failure "${SCRIPT_NAME}" "apt" "update" "update" "initial apt-get update failed"
+ die "initial apt-get update failed"
+ fi
+else
+ report "${SCRIPT_NAME}" "apt" "update" "update" "ok" "initial apt update (dry-run)" ""
+fi
+
+install_pkg_group base "${BASE_PACKAGES[@]}"
+case "${PROFILE}" in
+ minimal) ;;
+ core) install_pkg_group core "${CORE_PACKAGES[@]}" ;;
+ full)
+ install_pkg_group core "${CORE_PACKAGES[@]}"
+ install_pkg_group full-extra "${FULL_EXTRA_PACKAGES[@]}"
+ ;;
+esac
+
+normalize_service_baseline
+apply_hostname_locale_timezone
+
+if [[ ${USE_DB_APT_SOURCES} == true ]]; then
+ restore_apt_sources_from_db
+fi
+
+if [[ ${FROM_DB_MANUAL} == true ]]; then
+ install_from_db_manual
+fi
+
+install -d -m 0755 /srv /srv/www /srv/docs /etc/wireguard /etc/labunix /var/lib/labunix
+report "${SCRIPT_NAME}" "filesystem" "base-dirs" "create" "ok" "/srv /etc/wireguard /etc/labunix prepared" ""
+
+if [[ ${START_SERVICES} == true ]]; then
+ start_safe_services
+else
+ report "${SCRIPT_NAME}" "services" "safe-start" "start" "skipped" "--start-safe-services not requested" ""
+fi
+
+set_state install_base done
+report "${SCRIPT_NAME}" "run" "finish" "exit" "ok" "completed" ""
+log "${SCRIPT_NAME}: done; report=${REPORT_FILE}"
diff --git a/straper/rebuild.conf.example b/straper/rebuild.conf.example
new file mode 100644
index 0000000..ba1e9ab
--- /dev/null
+++ b/straper/rebuild.conf.example
@@ -0,0 +1,10 @@
+# Example: /etc/labunix/rebuild.conf
+DB_DIR=/srv/sanctum-rebuild
+ROLE=lab
+PROFILE=core
+DNS_MODE=auto
+NETWORK_MODE=safe
+START_SERVICES=false
+RESTORE_SECRETS=false
+RESTORE_IDENTITIES=false
+ASSUME_YES=false
diff --git a/straper/restore-configs.sh b/straper/restore-configs.sh
new file mode 100644
index 0000000..db81487
--- /dev/null
+++ b/straper/restore-configs.sh
@@ -0,0 +1,521 @@
+#!/usr/bin/env bash
+set -Eeuo pipefail
+
+SCRIPT_DIR="$(cd -- "$(dirname -- "$0")" && pwd)"
+# shellcheck disable=SC1091
+source "${SCRIPT_DIR}/lib/common.sh"
+
+SCRIPT_NAME="restore-configs.sh"
+CATEGORIES=()
+LIST_CATEGORIES=false
+
+usage() {
+ cat <<USAGE
+Usage: sudo ./restore-configs.sh [options]
+
+Purpose:
+ Restore selected configs from a capture DB in a controlled, category-based way.
+ Intended to be non-fatal per item whenever possible. Sensitive/identity-heavy
+ categories should be restored deliberately.
+
+Options:
+ --db-dir PATH
+ --role lab|hardware|replacement
+ --category NAME May be used multiple times
+ --list-categories
+ --yes Non-interactive yes to prompts
+ --dry-run
+ --verbose
+ --help
+
+Categories:
+ system-basics
+ users
+ ssh
+ network
+ dns
+ firewall
+ nginx
+ mariadb
+ postfix
+ prosody
+ tor
+ i2pd
+ docker
+ monitoring
+USAGE
+}
+
+while [[ $# -gt 0 ]]; do
+ case "$1" in
+ --db-dir) DB_DIR="$2"; PUBLIC_DIR="${DB_DIR}/db/public"; SECRET_DIR="${DB_DIR}/db/secret"; shift ;;
+ --role) ROLE="$2"; shift ;;
+ --category) CATEGORIES+=("$2"); shift ;;
+ --list-categories) LIST_CATEGORIES=true ;;
+ --yes) ASSUME_YES=true ;;
+ --dry-run) DRY_RUN=true ;;
+ --verbose) VERBOSE=true ;;
+ --help) usage; exit 0 ;;
+ *) die "unknown option: $1" ;;
+ esac
+ shift
+done
+
+if [[ "${LIST_CATEGORIES}" == true ]]; then
+ printf '%s\n' \
+ system-basics \
+ users \
+ ssh \
+ network \
+ dns \
+ firewall \
+ nginx \
+ mariadb \
+ postfix \
+ prosody \
+ tor \
+ i2pd \
+ docker \
+ monitoring
+ exit 0
+fi
+
+ensure_root
+load_optional_config
+ensure_runtime_dirs
+require_cmd cp chmod chown find grep sed awk systemctl
+
+[[ -d "${PUBLIC_DIR}" ]] || die "public DB not found: ${PUBLIC_DIR}"
+[[ ${#CATEGORIES[@]} -gt 0 ]] || die "no categories specified; use --category or --list-categories"
+
+log "${SCRIPT_NAME}: role=${ROLE} categories=${CATEGORIES[*]} db=${DB_DIR}"
+report "${SCRIPT_NAME}" "run" "start" "init" "ok" "role=${ROLE} categories=${CATEGORIES[*]}" ""
+
+case "${ROLE}" in
+ lab|hardware|replacement) ;;
+ *) die "invalid role: ${ROLE}" ;;
+esac
+
+fix_sudoers_perms() {
+ if [[ ${DRY_RUN} == true ]]; then
+ printf '[dry] normalize sudoers ownership and permissions\n'
+ return 0
+ fi
+
+ if [[ -f /etc/sudoers ]]; then
+ chown root:root /etc/sudoers || true
+ chmod 0440 /etc/sudoers || true
+ fi
+
+ if [[ -d /etc/sudoers.d ]]; then
+ chown root:root /etc/sudoers.d || true
+ chmod 0755 /etc/sudoers.d || true
+ find /etc/sudoers.d -maxdepth 1 -type f -exec chown root:root {} \; || true
+ find /etc/sudoers.d -maxdepth 1 -type f -exec chmod 0440 {} \; || true
+ fi
+
+ if visudo -c >/dev/null 2>&1; then
+ report "${SCRIPT_NAME}" "users" "sudoers-perms" "fix" "changed" "sudoers ownership and permissions normalized" ""
+ return 0
+ fi
+
+ note_failure "${SCRIPT_NAME}" "users" "sudoers-perms" "fix" "visudo validation failed after permission normalization"
+ return 1
+}
+
+maybe_restore() {
+ local category="$1" item="$2" src="$3" dst="$4"
+ local prompt_text="Restore ${category}/${item} -> ${dst}?"
+
+ if ! prompt_yes_no "${prompt_text}" yes; then
+ report "${SCRIPT_NAME}" "${category}" "${item}" "restore" "skipped" "operator skipped" ""
+ return 0
+ fi
+
+ restore_path "${SCRIPT_NAME}" "${category}" "${item}" "${src}" "${dst}" || true
+}
+
+category_enabled() {
+ local want="$1" x
+ for x in "${CATEGORIES[@]}"; do
+ [[ "${x}" == "${want}" ]] && return 0
+ done
+ return 1
+}
+
+restore_system_basics() {
+ local base="${PUBLIC_DIR}/system"
+ [[ -d "${base}" ]] || { mark_manual "${SCRIPT_NAME}" "system-basics" "db" "missing ${base}"; return 0; }
+
+ if ! prompt_yes_no "Restore category 'system-basics'?" yes; then
+ report "${SCRIPT_NAME}" "system-basics" "category" "restore" "skipped" "operator skipped category" ""
+ return 0
+ fi
+
+ maybe_restore "system-basics" "etc-hostname" "${base}/etc-hostname" "/etc/hostname"
+ maybe_restore "system-basics" "etc-hosts" "${base}/etc-hosts" "/etc/hosts"
+ maybe_restore "system-basics" "etc-environment" "${base}/etc-environment" "/etc/environment"
+ maybe_restore "system-basics" "locale.gen" "${base}/locale.gen" "/etc/locale.gen"
+ maybe_restore "system-basics" "timezone" "${base}/timezone" "/etc/timezone"
+
+ if [[ -f "${base}/etc-hostname" ]]; then
+ local captured_hostname
+ captured_hostname="$(head -n1 "${base}/etc-hostname" 2>/dev/null || true)"
+ if [[ -n "${captured_hostname}" ]]; then
+ if prompt_yes_no "Apply captured hostname now?" yes; then
+ if [[ ${DRY_RUN} == true ]]; then
+ printf '[dry] hostnamectl set-hostname %s\n' "${captured_hostname}"
+ else
+ hostnamectl set-hostname "${captured_hostname}" || note_failure "${SCRIPT_NAME}" "system-basics" "hostnamectl" "set" "hostnamectl failed"
+ fi
+ report "${SCRIPT_NAME}" "system-basics" "hostnamectl" "set" "changed" "${captured_hostname}" ""
+ else
+ report "${SCRIPT_NAME}" "system-basics" "hostnamectl" "set" "skipped" "operator skipped hostnamectl" ""
+ fi
+ fi
+ fi
+}
+
+restore_users() {
+ local base="${PUBLIC_DIR}/users"
+ [[ -d "${base}" ]] || { mark_manual "${SCRIPT_NAME}" "users" "db" "missing ${base}"; return 0; }
+
+ if ! prompt_yes_no "Restore category 'users'?" yes; then
+ report "${SCRIPT_NAME}" "users" "category" "restore" "skipped" "operator skipped category" ""
+ return 0
+ fi
+
+ maybe_restore "users" "sudoers" "${base}/sudoers" "/etc/sudoers"
+ maybe_restore "users" "sudoers.d" "${base}/sudoers.d" "/etc/sudoers.d"
+ fix_sudoers_perms || true
+ maybe_restore "users" "shells" "${base}/shells" "/etc/shells"
+ maybe_restore "users" "login.defs" "${base}/login.defs" "/etc/login.defs"
+}
+
+restore_ssh() {
+ local pub_users="${PUBLIC_DIR}/users"
+ local sec_ssh="${SECRET_DIR}/ssh"
+
+ if ! prompt_yes_no "Restore category 'ssh'?" yes; then
+ report "${SCRIPT_NAME}" "ssh" "category" "restore" "skipped" "operator skipped category" ""
+ return 0
+ fi
+
+ maybe_restore "ssh" "sshd_config" "${pub_users}/sshd_config" "/etc/ssh/sshd_config"
+ maybe_restore "ssh" "sshd_config.d" "${pub_users}/sshd_config.d" "/etc/ssh/sshd_config.d"
+
+ if [[ "${ROLE}" == "replacement" || "${ROLE}" == "hardware" ]]; then
+ if [[ -d "${sec_ssh}/etc-ssh" ]]; then
+ if prompt_yes_no "Restore SSH host keys from secret DB?" no; then
+ restore_path "${SCRIPT_NAME}" "ssh" "host-keys" "${sec_ssh}/etc-ssh" "/etc/ssh" || true
+ else
+ report "${SCRIPT_NAME}" "ssh" "host-keys" "restore" "skipped" "operator skipped host keys" ""
+ fi
+ fi
+ else
+ report "${SCRIPT_NAME}" "ssh" "host-keys" "restore" "skipped" "lab role: host keys not restored" ""
+ fi
+
+ if [[ -d "${sec_ssh}/user-lukasz" ]]; then
+ if prompt_yes_no "Restore lukasz user SSH material from secret DB?" no; then
+ restore_path "${SCRIPT_NAME}" "ssh" "user-lukasz" "${sec_ssh}/user-lukasz" "/home/lukasz/.ssh" || true
+ if [[ ${DRY_RUN} == false ]]; then
+ chown -R lukasz:lukasz /home/lukasz/.ssh 2>/dev/null || true
+ chmod 700 /home/lukasz/.ssh 2>/dev/null || true
+ find /home/lukasz/.ssh -maxdepth 1 -type f -exec chmod 600 {} \; 2>/dev/null || true
+ fi
+ else
+ report "${SCRIPT_NAME}" "ssh" "user-lukasz" "restore" "skipped" "operator skipped user ssh material" ""
+ fi
+ fi
+
+ if have_cmd sshd; then
+ if [[ ${DRY_RUN} == true ]]; then
+ printf '[dry] sshd -t\n'
+ report "${SCRIPT_NAME}" "ssh" "validate" "check" "ok" "dry-run only" ""
+ else
+ if sshd -t; then
+ report "${SCRIPT_NAME}" "ssh" "validate" "check" "ok" "sshd config validates" ""
+ systemctl restart ssh >/dev/null 2>&1 || true
+ else
+ note_failure "${SCRIPT_NAME}" "ssh" "validate" "check" "sshd validation failed"
+ fi
+ fi
+ fi
+}
+
+restore_network() {
+ local base="${PUBLIC_DIR}/network"
+ [[ -d "${base}" ]] || { mark_manual "${SCRIPT_NAME}" "network" "db" "missing ${base}"; return 0; }
+
+ if ! prompt_yes_no "Restore category 'network'? This can disrupt connectivity." no; then
+ report "${SCRIPT_NAME}" "network" "category" "restore" "skipped" "operator skipped category" ""
+ return 0
+ fi
+
+ if [[ "${ROLE}" == "lab" ]]; then
+ manual_line="lab role: full network restore intentionally skipped"
+ report "${SCRIPT_NAME}" "network" "category" "manual" "manual" "${manual_line}" ""
+ return 0
+ fi
+
+ maybe_restore "network" "etc-network" "${base}/etc-network" "/etc/network"
+ maybe_restore "network" "etc-netplan" "${base}/etc-netplan" "/etc/netplan"
+ maybe_restore "network" "systemd-network" "${base}/systemd-network" "/etc/systemd/network"
+ maybe_restore "network" "nsswitch.conf" "${base}/nsswitch.conf" "/etc/nsswitch.conf"
+ maybe_restore "network" "hosts.allow" "${base}/hosts.allow" "/etc/hosts.allow"
+ maybe_restore "network" "hosts.deny" "${base}/hosts.deny" "/etc/hosts.deny"
+}
+
+restore_dns() {
+ local base="${PUBLIC_DIR}/dns"
+ [[ -d "${base}" ]] || { mark_manual "${SCRIPT_NAME}" "dns" "db" "missing ${base}"; return 0; }
+
+ if ! prompt_yes_no "Restore category 'dns'? This can disrupt resolver state." no; then
+ report "${SCRIPT_NAME}" "dns" "category" "restore" "skipped" "operator skipped category" ""
+ return 0
+ fi
+
+ if [[ "${ROLE}" == "lab" ]]; then
+ report "${SCRIPT_NAME}" "dns" "category" "manual" "manual" "lab role: DNS restore intentionally skipped" ""
+ return 0
+ fi
+
+ maybe_restore "dns" "dnsmasq.conf" "${base}/dnsmasq.conf" "/etc/dnsmasq.conf"
+ maybe_restore "dns" "dnsmasq.d" "${base}/dnsmasq.d" "/etc/dnsmasq.d"
+ maybe_restore "dns" "etc-unbound" "${base}/etc-unbound" "/etc/unbound"
+ maybe_restore "dns" "resolv.conf" "${base}/resolv.conf" "/etc/resolv.conf"
+}
+
+restore_firewall() {
+ local base="${PUBLIC_DIR}/firewall"
+ [[ -d "${base}" ]] || { mark_manual "${SCRIPT_NAME}" "firewall" "db" "missing ${base}"; return 0; }
+
+ if ! prompt_yes_no "Restore category 'firewall'? This can disrupt connectivity." no; then
+ report "${SCRIPT_NAME}" "firewall" "category" "restore" "skipped" "operator skipped category" ""
+ return 0
+ fi
+
+ if [[ "${ROLE}" == "lab" ]]; then
+ report "${SCRIPT_NAME}" "firewall" "category" "manual" "manual" "lab role: firewall restore intentionally skipped" ""
+ return 0
+ fi
+
+ maybe_restore "firewall" "nftables.conf" "${base}/nftables.conf" "/etc/nftables.conf"
+ maybe_restore "firewall" "nftables.d" "${base}/nftables.d" "/etc/nftables.d"
+
+ if [[ ${DRY_RUN} == false && -f /etc/nftables.conf ]] && have_cmd nft; then
+ nft -c -f /etc/nftables.conf >/dev/null 2>&1 || note_failure "${SCRIPT_NAME}" "firewall" "validate" "check" "nftables config validation failed"
+ fi
+}
+
+restore_nginx() {
+ local base="${PUBLIC_DIR}/nginx/etc-nginx"
+ [[ -d "${base}" ]] || { mark_manual "${SCRIPT_NAME}" "nginx" "db" "missing ${base}"; return 0; }
+
+ if ! prompt_yes_no "Restore category 'nginx'?" no; then
+ report "${SCRIPT_NAME}" "nginx" "category" "restore" "skipped" "operator skipped category" ""
+ return 0
+ fi
+
+ maybe_restore "nginx" "etc-nginx" "${base}" "/etc/nginx"
+
+ if [[ "${ROLE}" == "lab" ]]; then
+ if [[ ${DRY_RUN} == true ]]; then
+ printf '[dry] sanitize /etc/nginx for lab role\n'
+ printf '[dry] rm -rf /etc/nginx/nginx\n'
+ printf '[dry] find /etc/nginx -maxdepth 1 -type d -name '\''sites-available.bak.*'\'' -exec rm -rf {} +\n'
+ printf '[dry] rm -f /etc/nginx/sites-enabled/*\n'
+ printf '[dry] ln -sf ../sites-available/default /etc/nginx/sites-enabled/default\n'
+ else
+ rm -rf /etc/nginx/nginx 2>/dev/null || true
+ find /etc/nginx -maxdepth 1 -type d -name 'sites-available.bak.*' -exec rm -rf {} + 2>/dev/null || true
+
+ mkdir -p /etc/nginx/sites-enabled
+ find /etc/nginx/sites-enabled -mindepth 1 -maxdepth 1 -exec rm -f {} + 2>/dev/null || true
+
+ if [[ -e /etc/nginx/sites-available/default ]]; then
+ ln -sf ../sites-available/default /etc/nginx/sites-enabled/default
+ fi
+ fi
+
+ report "${SCRIPT_NAME}" "nginx" "lab-sanitize" "restore" "changed" \
+ "lab role: dropped captured production sites-enabled and restored only default site" ""
+ fi
+}
+
+restore_mariadb() {
+ local base="${PUBLIC_DIR}/mariadb/etc-mysql"
+ [[ -d "${base}" ]] || { mark_manual "${SCRIPT_NAME}" "mariadb" "db" "missing ${base}"; return 0; }
+
+ if ! prompt_yes_no "Restore category 'mariadb'?" no; then
+ report "${SCRIPT_NAME}" "mariadb" "category" "restore" "skipped" "operator skipped category" ""
+ return 0
+ fi
+
+ maybe_restore "mariadb" "etc-mysql" "${base}" "/etc/mysql"
+}
+
+restore_postfix() {
+ local base="${PUBLIC_DIR}/postfix"
+ [[ -d "${base}" ]] || {
+ mark_manual "${SCRIPT_NAME}" "postfix" "db" "missing ${base}"
+ return 0
+ }
+
+ if ! prompt_yes_no "Restore category 'postfix'?" no; then
+ report "${SCRIPT_NAME}" "postfix" "category" "restore" "skipped" "operator skipped category" ""
+ return 0
+ fi
+
+ if prompt_yes_no "Restore postfix/main.cf -> /etc/postfix/main.cf?" yes; then
+ restore_path "${SCRIPT_NAME}" "postfix" "main.cf" \
+ "${base}/main.cf" "/etc/postfix/main.cf" \
+ 0644 root root || true
+ else
+ report "${SCRIPT_NAME}" "postfix" "main.cf" "restore" "skipped" "operator skipped" ""
+ fi
+
+ if prompt_yes_no "Restore postfix/master.cf -> /etc/postfix/master.cf?" yes; then
+ restore_path "${SCRIPT_NAME}" "postfix" "master.cf" \
+ "${base}/master.cf" "/etc/postfix/master.cf" \
+ 0644 root root || true
+ else
+ report "${SCRIPT_NAME}" "postfix" "master.cf" "restore" "skipped" "operator skipped" ""
+ fi
+}
+
+restore_prosody() {
+ local pub_base="${PUBLIC_DIR}/prosody/etc-prosody"
+ local sec_base="${SECRET_DIR}/prosody/etc-prosody"
+ local base=""
+ local label=""
+
+ if ! prompt_yes_no "Restore category 'prosody'?" no; then
+ report "${SCRIPT_NAME}" "prosody" "category" "restore" "skipped" "operator skipped category" ""
+ return 0
+ fi
+
+ if [[ -d "${sec_base}" ]]; then
+ base="${sec_base}"
+ label="etc-prosody(secret)"
+ elif [[ -d "${pub_base}" ]]; then
+ base="${pub_base}"
+ label="etc-prosody(public)"
+ else
+ mark_manual "${SCRIPT_NAME}" "prosody" "db" "missing public/secret prosody config"
+ return 0
+ fi
+
+ if prompt_yes_no "Restore prosody/${label} -> /etc/prosody?" yes; then
+ restore_path "${SCRIPT_NAME}" "prosody" "${label}" "${base}" "/etc/prosody" || true
+
+ if [[ ${DRY_RUN} == true ]]; then
+ printf '[dry] chown -R root:root /etc/prosody\n'
+ printf '[dry] find /etc/prosody -type d -exec chmod 0755 {} +\n'
+ printf '[dry] find /etc/prosody -type f -exec chmod 0644 {} +\n'
+ printf '[dry] find /etc/prosody/certs -type f -exec chmod 0640 {} + 2>/dev/null || true\n'
+ else
+ chown -R root:root /etc/prosody 2>/dev/null || true
+ find /etc/prosody -type d -exec chmod 0755 {} + 2>/dev/null || true
+ find /etc/prosody -type f -exec chmod 0644 {} + 2>/dev/null || true
+ [[ -d /etc/prosody/certs ]] && find /etc/prosody/certs -type f -exec chmod 0640 {} + 2>/dev/null || true
+ fi
+
+ report "${SCRIPT_NAME}" "prosody" "metadata" "restore" "changed" \
+ "normalized /etc/prosody ownership=root:root dirs=0755 files=0644 certs=0640" ""
+ else
+ report "${SCRIPT_NAME}" "prosody" "${label}" "restore" "skipped" "operator skipped" ""
+ fi
+}
+
+restore_tor() {
+ local pub_base="${PUBLIC_DIR}/tor"
+ local sec_base="${SECRET_DIR}/tor"
+
+ if ! prompt_yes_no "Restore category 'tor'?" no; then
+ report "${SCRIPT_NAME}" "tor" "category" "restore" "skipped" "operator skipped category" ""
+ return 0
+ fi
+
+ maybe_restore "tor" "torrc" "${pub_base}/torrc" "/etc/tor/torrc"
+ maybe_restore "tor" "torrc.d" "${pub_base}/torrc.d" "/etc/tor/torrc.d"
+
+ if [[ "${ROLE}" == "replacement" && -d "${sec_base}/var-lib-tor" ]]; then
+ maybe_restore "tor" "var-lib-tor" "${sec_base}/var-lib-tor" "/var/lib/tor"
+ else
+ report "${SCRIPT_NAME}" "tor" "identity" "restore" "skipped" "tor private data not restored in this role" ""
+ fi
+}
+
+restore_i2pd() {
+ local pub_base="${PUBLIC_DIR}/i2pd"
+ local sec_base="${SECRET_DIR}/i2pd"
+
+ if ! prompt_yes_no "Restore category 'i2pd'?" no; then
+ report "${SCRIPT_NAME}" "i2pd" "category" "restore" "skipped" "operator skipped category" ""
+ return 0
+ fi
+
+ maybe_restore "i2pd" "etc-i2pd" "${pub_base}/etc-i2pd" "/etc/i2pd"
+
+ if [[ "${ROLE}" == "replacement" && -d "${sec_base}/var-lib-i2pd" ]]; then
+ maybe_restore "i2pd" "var-lib-i2pd" "${sec_base}/var-lib-i2pd" "/var/lib/i2pd"
+ else
+ report "${SCRIPT_NAME}" "i2pd" "identity" "restore" "skipped" "i2pd private data not restored in this role" ""
+ fi
+}
+
+restore_docker() {
+ local pub_base="${PUBLIC_DIR}/docker"
+ local sec_base="${SECRET_DIR}/docker"
+
+ if ! prompt_yes_no "Restore category 'docker'?" no; then
+ report "${SCRIPT_NAME}" "docker" "category" "restore" "skipped" "operator skipped category" ""
+ return 0
+ fi
+
+ maybe_restore "docker" "daemon.json" "${pub_base}/daemon.json" "/etc/docker/daemon.json"
+
+ if [[ -d "${sec_base}/compose-full" ]]; then
+ report "${SCRIPT_NAME}" "docker" "compose-full" "manual" "manual" "compose files available in secret DB; restore manually per stack" ""
+ fi
+}
+
+restore_monitoring() {
+ if ! prompt_yes_no "Restore category 'monitoring'?" no; then
+ report "${SCRIPT_NAME}" "monitoring" "category" "restore" "skipped" "operator skipped category" ""
+ return 0
+ fi
+
+ [[ -d "${PUBLIC_DIR}/prometheus/etc-prometheus" ]] && maybe_restore "monitoring" "prometheus" "${PUBLIC_DIR}/prometheus/etc-prometheus" "/etc/prometheus"
+ [[ -f "${PUBLIC_DIR}/prometheus/node-exporter-defaults" ]] && maybe_restore "monitoring" "prometheus-node-exporter" "${PUBLIC_DIR}/prometheus/node-exporter-defaults" "/etc/default/prometheus-node-exporter"
+ [[ -d "${PUBLIC_DIR}/loki/etc-loki" ]] && maybe_restore "monitoring" "loki" "${PUBLIC_DIR}/loki/etc-loki" "/etc/loki"
+ [[ -d "${PUBLIC_DIR}/grafana/etc-grafana" ]] && maybe_restore "monitoring" "grafana" "${PUBLIC_DIR}/grafana/etc-grafana" "/etc/grafana"
+ [[ -d "${SECRET_DIR}/alloy/etc-alloy" ]] && maybe_restore "monitoring" "alloy" "${SECRET_DIR}/alloy/etc-alloy" "/etc/alloy"
+}
+
+for category in "${CATEGORIES[@]}"; do
+ log "processing category: ${category}"
+ case "${category}" in
+ system-basics) restore_system_basics ;;
+ users) restore_users ;;
+ ssh) restore_ssh ;;
+ network) restore_network ;;
+ dns) restore_dns ;;
+ firewall) restore_firewall ;;
+ nginx) restore_nginx ;;
+ mariadb) restore_mariadb ;;
+ postfix) restore_postfix ;;
+ prosody) restore_prosody ;;
+ tor) restore_tor ;;
+ i2pd) restore_i2pd ;;
+ docker) restore_docker ;;
+ monitoring) restore_monitoring ;;
+ *) note_failure "${SCRIPT_NAME}" "category" "${category}" "restore" "unknown category" ;;
+ esac
+done
+
+set_state restore_configs done
+report "${SCRIPT_NAME}" "run" "finish" "exit" "ok" "completed" ""
+log "${SCRIPT_NAME}: done; report=${REPORT_FILE}"
diff --git a/straper/sanctum-rebuild-toolkit.tar.gz b/straper/sanctum-rebuild-toolkit.tar.gz
new file mode 100644
index 0000000..5a49146
--- /dev/null
+++ b/straper/sanctum-rebuild-toolkit.tar.gz
Binary files differ