From 1f1167338a2a33fd5436b6dc56c31d818bb834f3 Mon Sep 17 00:00:00 2001 From: Stefan Davis Date: Thu, 4 Sep 2025 04:36:13 +1000 Subject: [PATCH 1/9] pypy build test --- debian/control | 9 ++++----- debian/rules | 15 +++++++++++++-- debian/tklbam.install | 2 ++ duct-tape-deps.sh | 34 ++++++++++++++++++++++++++++++++++ 4 files changed, 53 insertions(+), 7 deletions(-) create mode 100755 duct-tape-deps.sh diff --git a/debian/control b/debian/control index df6ebf9..68b94e7 100644 --- a/debian/control +++ b/debian/control @@ -5,10 +5,11 @@ Maintainer: Liraz Siri Build-Depends: debhelper (>= 10), dh-exec, - dh-python2 (>= 4.20201102+turnkey0), - python-all (>= 2.6.6-3~), + libcurl4-openssl-dev, + wget, + tar, + libssl-dev Standards-Version: 4.0.0 -X-Python-Version: >= 2.6 Package: tklbam Architecture: all @@ -20,8 +21,6 @@ Depends: pycurl-wrapper (>= 1.2), pycurl-wrapper (<< 2.0), gpg (>= 2.2.12), - ${python:Depends}, - python-crypto, python-simplejson, tklbam-duplicity (>= 0.6.18), tklbam-python-boto (>= 2.3.0-2turnkey), diff --git a/debian/rules b/debian/rules index 500f8be..5ed7b2f 100755 --- a/debian/rules +++ b/debian/rules @@ -1,9 +1,20 @@ #! /usr/bin/make -f -include /usr/share/dpkg/pkg-info.mk +progname=$(shell awk '/^Source/ {print $$2}' debian/control) +buildroot=debian/$(progname) + +deproot=$(progname)/deproot +tmp=debian/tmp %: - dh $@ --with python2 --buildsystem=makefile + dh $@ + +all: + +override_dh_auto_build: + mkdir -p $(tmp) $(deproot) + DEPROOT=$(deproot) ./duct-tape-deps.sh + rm -rf $(tmp) override_dh_auto_install: dh_auto_install -- prefix=debian/$(DEB_SOURCE)/usr diff --git a/debian/tklbam.install b/debian/tklbam.install index 28a5861..b0aa952 100755 --- a/debian/tklbam.install +++ b/debian/tklbam.install @@ -10,3 +10,5 @@ contrib/example-hook => /etc/tklbam/hooks.d/example contrib/example-hook-pre-backup => /etc/tklbam/hooks.d/example-pre-backup contrib/example-hook-post-restore => /etc/tklbam/hooks.d/example-post-restore contrib/ez-apt-install.sh => /usr/share/tklbam/contrib/ez-apt-install.sh + +deproot => /usr/lib/tklbam/deps diff --git a/duct-tape-deps.sh b/duct-tape-deps.sh new file mode 100755 index 0000000..8f5618d --- /dev/null +++ b/duct-tape-deps.sh @@ -0,0 +1,34 @@ +#!/bin/bash + +pypy_root='pypy2.7-v7.3.20-linux64' +pypy_archive="$pypy_root.tar.bz2" + +DEPROOT="$(pwd)/debian/tmp/dh-*/deproot" + +mkdir -p $DEPROOT /usr/lib/tklbam/deps + +cd debian/tmp + +wget "https://downloads.python.org/pypy/$pypy_archive" +tar -xvf "$pypy_archive" + +pypy_root="$(pwd)/$pypy_root" + +rm "$pypy_archive" + +git clone https://github.com/turnkeylinux/python-crypto +cd python-crypto +LD_LIBRARY_PATH="$LD_LIBRARY_PATH:$pypy_root/bin" "$pypy_root/bin/python2" setup.py build + +cd .. +git clone https://github.com/turnkeylinux/python-pycurl +cd python-pycurl +LD_LIBRARY_PATH="$LD_LIBRARY_PATH:$pypy_root/bin" "$pypy_root/bin/python2" setup.py build + +cd .. +mv pypy*/* $DEPROOT +mv python-crypto/build/lib*/* $DEPROOT/site-packages +rm -rf python-crypto +mv python-pycurl/build/lib*/* $DEPROOT/site-packages +rm -rf python-pycurl +cd .. From f250c00760d5fda0c6d2536ba3106a74db006dc4 Mon Sep 17 00:00:00 2001 From: Jeremy Davis Date: Wed, 17 Sep 2025 07:21:30 +0000 Subject: [PATCH 2/9] Move tklbam source files, remove Makefile --- Makefile | 96 ------------------- backup.py => lib/backup.py | 0 changes.py => lib/changes.py | 0 cliwrapper.py => lib/cliwrapper.py | 0 cmd.py => lib/cmd.py | 0 cmd_backup.py => lib/cmd_backup.py | 0 cmd_escrow.py => lib/cmd_escrow.py | 0 cmd_init.py => lib/cmd_init.py | 0 cmd_internal.py => lib/cmd_internal.py | 0 .../cmd_internals}/__init__.py | 0 .../cmd_internals}/cmd_create_profile.py | 0 .../cmd_internals}/cmd_delete.py | 0 .../cmd_internals}/cmd_detect_profile_id.py | 0 .../cmd_internals}/cmd_dirindex.py | 0 .../cmd_internals}/cmd_fixstat.py | 0 .../cmd_internals}/cmd_fs2mysql.py | 0 .../cmd_internals}/cmd_fs2pgsql.py | 0 .../cmd_internals}/cmd_merge_userdb.py | 0 .../cmd_internals}/cmd_mysql2fs.py | 0 .../cmd_internals}/cmd_newpkgs.py | 0 .../cmd_internals}/cmd_newpkgs_install.py | 0 .../cmd_internals}/cmd_pgsql2fs.py | 0 .../cmd_internals}/cmd_stsagent.py | 0 cmd_list.py => lib/cmd_list.py | 0 cmd_passphrase.py => lib/cmd_passphrase.py | 0 cmd_restore.py => lib/cmd_restore.py | 0 .../cmd_restore_rollback.py | 0 cmd_status.py => lib/cmd_status.py | 0 conf.py => lib/conf.py | 0 dblimits.py => lib/dblimits.py | 0 dirindex.py => lib/dirindex.py | 0 dummyhub.py => lib/dummyhub.py | 0 duplicity.py => lib/duplicity.py | 0 hooks.py => lib/hooks.py | 0 hub.py => lib/hub.py | 0 keypacket.py => lib/keypacket.py | 0 mysql.py => lib/mysql.py | 0 passphrase.py => lib/passphrase.py | 0 pathmap.py => lib/pathmap.py | 0 pgsql.py => lib/pgsql.py | 0 pkgman.py => lib/pkgman.py | 0 registry.py => lib/registry.py | 0 restore.py => lib/restore.py | 0 retry.py => lib/retry.py | 0 rollback.py => lib/rollback.py | 0 squid.py => lib/squid.py | 0 userdb.py => lib/userdb.py | 0 utils.py => lib/utils.py | 0 version.py => lib/version.py | 0 49 files changed, 96 deletions(-) delete mode 100644 Makefile rename backup.py => lib/backup.py (100%) rename changes.py => lib/changes.py (100%) rename cliwrapper.py => lib/cliwrapper.py (100%) rename cmd.py => lib/cmd.py (100%) rename cmd_backup.py => lib/cmd_backup.py (100%) rename cmd_escrow.py => lib/cmd_escrow.py (100%) rename cmd_init.py => lib/cmd_init.py (100%) rename cmd_internal.py => lib/cmd_internal.py (100%) rename {cmd_internals => lib/cmd_internals}/__init__.py (100%) rename {cmd_internals => lib/cmd_internals}/cmd_create_profile.py (100%) rename {cmd_internals => lib/cmd_internals}/cmd_delete.py (100%) rename {cmd_internals => lib/cmd_internals}/cmd_detect_profile_id.py (100%) rename {cmd_internals => lib/cmd_internals}/cmd_dirindex.py (100%) rename {cmd_internals => lib/cmd_internals}/cmd_fixstat.py (100%) rename {cmd_internals => lib/cmd_internals}/cmd_fs2mysql.py (100%) rename {cmd_internals => lib/cmd_internals}/cmd_fs2pgsql.py (100%) rename {cmd_internals => lib/cmd_internals}/cmd_merge_userdb.py (100%) rename {cmd_internals => lib/cmd_internals}/cmd_mysql2fs.py (100%) rename {cmd_internals => lib/cmd_internals}/cmd_newpkgs.py (100%) rename {cmd_internals => lib/cmd_internals}/cmd_newpkgs_install.py (100%) rename {cmd_internals => lib/cmd_internals}/cmd_pgsql2fs.py (100%) rename {cmd_internals => lib/cmd_internals}/cmd_stsagent.py (100%) rename cmd_list.py => lib/cmd_list.py (100%) rename cmd_passphrase.py => lib/cmd_passphrase.py (100%) rename cmd_restore.py => lib/cmd_restore.py (100%) rename cmd_restore_rollback.py => lib/cmd_restore_rollback.py (100%) rename cmd_status.py => lib/cmd_status.py (100%) rename conf.py => lib/conf.py (100%) rename dblimits.py => lib/dblimits.py (100%) rename dirindex.py => lib/dirindex.py (100%) rename dummyhub.py => lib/dummyhub.py (100%) rename duplicity.py => lib/duplicity.py (100%) rename hooks.py => lib/hooks.py (100%) rename hub.py => lib/hub.py (100%) rename keypacket.py => lib/keypacket.py (100%) rename mysql.py => lib/mysql.py (100%) rename passphrase.py => lib/passphrase.py (100%) rename pathmap.py => lib/pathmap.py (100%) rename pgsql.py => lib/pgsql.py (100%) rename pkgman.py => lib/pkgman.py (100%) rename registry.py => lib/registry.py (100%) rename restore.py => lib/restore.py (100%) rename retry.py => lib/retry.py (100%) rename rollback.py => lib/rollback.py (100%) rename squid.py => lib/squid.py (100%) rename userdb.py => lib/userdb.py (100%) rename utils.py => lib/utils.py (100%) rename version.py => lib/version.py (100%) diff --git a/Makefile b/Makefile deleted file mode 100644 index b8f27a7..0000000 --- a/Makefile +++ /dev/null @@ -1,96 +0,0 @@ -# standard Python project Makefile -progname = $(shell awk '/^Source/ {print $$2}' debian/control) -name= - -prefix = /usr/local -PATH_BIN = $(prefix)/bin -PATH_INSTALL_LIB = $(prefix)/lib/$(progname) -PATH_DIST := $(progname)-$$(autoversion HEAD) - -SHELL = /bin/bash - -all: help - -debug: - $(foreach v, $V, $(warning $v = $($v))) - @true - -dist: - -mkdir -p $(PATH_DIST) - - cp -a .git $(PATH_DIST) - cd $(PATH_DIST) && git-checkout --force HEAD - - tar jcvf $(PATH_DIST).tar.bz2 $(PATH_DIST) - rm -rf $(PATH_DIST) - -#docs: -# $(MAKE) -C docs - -help: - @echo '=== Targets:' - @echo 'install [ prefix=path/to/usr ] # default: prefix=$(value prefix)' - @echo 'uninstall [ prefix=path/to/usr ]' - @echo 'docs' - @echo - @echo 'clean' - -# DRY macros -debbuild=debian/$(shell awk '/^Package/ {print $$2}' debian/control) - -truepath = $(shell echo $1 | sed -e 's|^$(debbuild)||') -libpath = $(call truepath,$(PATH_INSTALL_LIB))/$$(basename $1) -subcommand = $(progname)-$$(echo $1 | sed 's|.*/||; s/^cmd_//; s/_/-/g; s/.py$$//') -echo-do = echo $1; $1 - -# first argument: code we execute if there is just one executable module -# second argument: code we execute if there is more than on executable module -define with-py-executables - @modules=$$(find -maxdepth 1 -type f -name 'cmd_*.py' -perm -100); \ - modules_len=$$(echo $$modules | wc -w); \ - if [ $$modules_len = 1 ]; then \ - module=$$modules; \ - $(call echo-do, $1); \ - elif [ $$modules_len -gt 1 ]; then \ - for module in $$modules; do \ - $(call echo-do, $2); \ - done; \ - fi; -endef - -install: #docs - @echo - @echo \*\* CONFIG: prefix = $(prefix) \*\* - @echo - - install -d $(PATH_BIN) $(PATH_INSTALL_LIB) - cp *.py $(PATH_INSTALL_LIB) - - install -d $(PATH_INSTALL_LIB)/cmd_internals - cp cmd_internals/*.py $(PATH_INSTALL_LIB)/cmd_internals - - $(call with-py-executables, \ - ln -fs $(call libpath, $$module) $(PATH_BIN)/$(progname), \ - ln -fs $(call libpath, $$module) $(PATH_BIN)/$(call subcommand, $$module)) - - ln -fs $(call libpath, cmd.py) $(PATH_BIN)/$(progname) - -uninstall: - rm -rf $(PATH_INSTALL_LIB) - - $(call with-py-executables, \ - rm -f $(PATH_BIN)/$(progname), \ - rm -f $(PATH_BIN)/$(call subcommand, $$module)) - -clean: - $(MAKE) -C docs clean - - rm -f {,*/}*.pyc {,*/}*.pyo - - @emptydirs=$$(find -type d -empty); \ - if [ "$$emptydirs" ]; then \ - echo rmdir $$emptydirs; \ - rmdir $$emptydirs; \ - fi - -.PHONY: all debug dist docs help install uninstall clean diff --git a/backup.py b/lib/backup.py similarity index 100% rename from backup.py rename to lib/backup.py diff --git a/changes.py b/lib/changes.py similarity index 100% rename from changes.py rename to lib/changes.py diff --git a/cliwrapper.py b/lib/cliwrapper.py similarity index 100% rename from cliwrapper.py rename to lib/cliwrapper.py diff --git a/cmd.py b/lib/cmd.py similarity index 100% rename from cmd.py rename to lib/cmd.py diff --git a/cmd_backup.py b/lib/cmd_backup.py similarity index 100% rename from cmd_backup.py rename to lib/cmd_backup.py diff --git a/cmd_escrow.py b/lib/cmd_escrow.py similarity index 100% rename from cmd_escrow.py rename to lib/cmd_escrow.py diff --git a/cmd_init.py b/lib/cmd_init.py similarity index 100% rename from cmd_init.py rename to lib/cmd_init.py diff --git a/cmd_internal.py b/lib/cmd_internal.py similarity index 100% rename from cmd_internal.py rename to lib/cmd_internal.py diff --git a/cmd_internals/__init__.py b/lib/cmd_internals/__init__.py similarity index 100% rename from cmd_internals/__init__.py rename to lib/cmd_internals/__init__.py diff --git a/cmd_internals/cmd_create_profile.py b/lib/cmd_internals/cmd_create_profile.py similarity index 100% rename from cmd_internals/cmd_create_profile.py rename to lib/cmd_internals/cmd_create_profile.py diff --git a/cmd_internals/cmd_delete.py b/lib/cmd_internals/cmd_delete.py similarity index 100% rename from cmd_internals/cmd_delete.py rename to lib/cmd_internals/cmd_delete.py diff --git a/cmd_internals/cmd_detect_profile_id.py b/lib/cmd_internals/cmd_detect_profile_id.py similarity index 100% rename from cmd_internals/cmd_detect_profile_id.py rename to lib/cmd_internals/cmd_detect_profile_id.py diff --git a/cmd_internals/cmd_dirindex.py b/lib/cmd_internals/cmd_dirindex.py similarity index 100% rename from cmd_internals/cmd_dirindex.py rename to lib/cmd_internals/cmd_dirindex.py diff --git a/cmd_internals/cmd_fixstat.py b/lib/cmd_internals/cmd_fixstat.py similarity index 100% rename from cmd_internals/cmd_fixstat.py rename to lib/cmd_internals/cmd_fixstat.py diff --git a/cmd_internals/cmd_fs2mysql.py b/lib/cmd_internals/cmd_fs2mysql.py similarity index 100% rename from cmd_internals/cmd_fs2mysql.py rename to lib/cmd_internals/cmd_fs2mysql.py diff --git a/cmd_internals/cmd_fs2pgsql.py b/lib/cmd_internals/cmd_fs2pgsql.py similarity index 100% rename from cmd_internals/cmd_fs2pgsql.py rename to lib/cmd_internals/cmd_fs2pgsql.py diff --git a/cmd_internals/cmd_merge_userdb.py b/lib/cmd_internals/cmd_merge_userdb.py similarity index 100% rename from cmd_internals/cmd_merge_userdb.py rename to lib/cmd_internals/cmd_merge_userdb.py diff --git a/cmd_internals/cmd_mysql2fs.py b/lib/cmd_internals/cmd_mysql2fs.py similarity index 100% rename from cmd_internals/cmd_mysql2fs.py rename to lib/cmd_internals/cmd_mysql2fs.py diff --git a/cmd_internals/cmd_newpkgs.py b/lib/cmd_internals/cmd_newpkgs.py similarity index 100% rename from cmd_internals/cmd_newpkgs.py rename to lib/cmd_internals/cmd_newpkgs.py diff --git a/cmd_internals/cmd_newpkgs_install.py b/lib/cmd_internals/cmd_newpkgs_install.py similarity index 100% rename from cmd_internals/cmd_newpkgs_install.py rename to lib/cmd_internals/cmd_newpkgs_install.py diff --git a/cmd_internals/cmd_pgsql2fs.py b/lib/cmd_internals/cmd_pgsql2fs.py similarity index 100% rename from cmd_internals/cmd_pgsql2fs.py rename to lib/cmd_internals/cmd_pgsql2fs.py diff --git a/cmd_internals/cmd_stsagent.py b/lib/cmd_internals/cmd_stsagent.py similarity index 100% rename from cmd_internals/cmd_stsagent.py rename to lib/cmd_internals/cmd_stsagent.py diff --git a/cmd_list.py b/lib/cmd_list.py similarity index 100% rename from cmd_list.py rename to lib/cmd_list.py diff --git a/cmd_passphrase.py b/lib/cmd_passphrase.py similarity index 100% rename from cmd_passphrase.py rename to lib/cmd_passphrase.py diff --git a/cmd_restore.py b/lib/cmd_restore.py similarity index 100% rename from cmd_restore.py rename to lib/cmd_restore.py diff --git a/cmd_restore_rollback.py b/lib/cmd_restore_rollback.py similarity index 100% rename from cmd_restore_rollback.py rename to lib/cmd_restore_rollback.py diff --git a/cmd_status.py b/lib/cmd_status.py similarity index 100% rename from cmd_status.py rename to lib/cmd_status.py diff --git a/conf.py b/lib/conf.py similarity index 100% rename from conf.py rename to lib/conf.py diff --git a/dblimits.py b/lib/dblimits.py similarity index 100% rename from dblimits.py rename to lib/dblimits.py diff --git a/dirindex.py b/lib/dirindex.py similarity index 100% rename from dirindex.py rename to lib/dirindex.py diff --git a/dummyhub.py b/lib/dummyhub.py similarity index 100% rename from dummyhub.py rename to lib/dummyhub.py diff --git a/duplicity.py b/lib/duplicity.py similarity index 100% rename from duplicity.py rename to lib/duplicity.py diff --git a/hooks.py b/lib/hooks.py similarity index 100% rename from hooks.py rename to lib/hooks.py diff --git a/hub.py b/lib/hub.py similarity index 100% rename from hub.py rename to lib/hub.py diff --git a/keypacket.py b/lib/keypacket.py similarity index 100% rename from keypacket.py rename to lib/keypacket.py diff --git a/mysql.py b/lib/mysql.py similarity index 100% rename from mysql.py rename to lib/mysql.py diff --git a/passphrase.py b/lib/passphrase.py similarity index 100% rename from passphrase.py rename to lib/passphrase.py diff --git a/pathmap.py b/lib/pathmap.py similarity index 100% rename from pathmap.py rename to lib/pathmap.py diff --git a/pgsql.py b/lib/pgsql.py similarity index 100% rename from pgsql.py rename to lib/pgsql.py diff --git a/pkgman.py b/lib/pkgman.py similarity index 100% rename from pkgman.py rename to lib/pkgman.py diff --git a/registry.py b/lib/registry.py similarity index 100% rename from registry.py rename to lib/registry.py diff --git a/restore.py b/lib/restore.py similarity index 100% rename from restore.py rename to lib/restore.py diff --git a/retry.py b/lib/retry.py similarity index 100% rename from retry.py rename to lib/retry.py diff --git a/rollback.py b/lib/rollback.py similarity index 100% rename from rollback.py rename to lib/rollback.py diff --git a/squid.py b/lib/squid.py similarity index 100% rename from squid.py rename to lib/squid.py diff --git a/userdb.py b/lib/userdb.py similarity index 100% rename from userdb.py rename to lib/userdb.py diff --git a/utils.py b/lib/utils.py similarity index 100% rename from utils.py rename to lib/utils.py diff --git a/version.py b/lib/version.py similarity index 100% rename from version.py rename to lib/version.py From 75c500c79a74991531f0d5390fc537ae7b9e53e0 Mon Sep 17 00:00:00 2001 From: Jeremy Davis Date: Wed, 17 Sep 2025 07:24:24 +0000 Subject: [PATCH 3/9] update debian/install & add debian/links to handle tklbam files & update debian/rules --- debian/rules | 10 +++------- debian/tklbam.install | 22 ++++++++++------------ debian/tklbam.links | 10 ++++++++++ 3 files changed, 23 insertions(+), 19 deletions(-) mode change 100755 => 100644 debian/tklbam.install create mode 100644 debian/tklbam.links diff --git a/debian/rules b/debian/rules index 5ed7b2f..50c48b1 100755 --- a/debian/rules +++ b/debian/rules @@ -9,12 +9,8 @@ tmp=debian/tmp %: dh $@ -all: - override_dh_auto_build: - mkdir -p $(tmp) $(deproot) - DEPROOT=$(deproot) ./duct-tape-deps.sh - rm -rf $(tmp) + ./duct-tape-deps.sh -override_dh_auto_install: - dh_auto_install -- prefix=debian/$(DEB_SOURCE)/usr +override_dh_shlibdeps: +override_dh_makeshlibs: diff --git a/debian/tklbam.install b/debian/tklbam.install old mode 100755 new mode 100644 index b0aa952..1de7eb1 --- a/debian/tklbam.install +++ b/debian/tklbam.install @@ -1,14 +1,12 @@ -#!/usr/bin/dh-exec +contrib/example-conf /etc/tklbam/conf +contrib/example-overrides /etc/tklbam/overrides +contrib/fixclock-hook /etc/tklbam/hooks.d/fixclock +contrib/maria-db-changes-hook /etc/tklbam/hooks.d/maria-db-changes +contrib/cron.sh /etc/cron.daily/tklbam-backup -contrib/example-conf => /etc/tklbam/conf -contrib/example-overrides => /etc/tklbam/overrides -contrib/fixclock-hook => /etc/tklbam/hooks.d/fixclock -contrib/maria-db-changes-hook => /etc/tklbam/hooks.d/maria-db-changes -contrib/cron.sh => /etc/cron.daily/tklbam-backup +contrib/example-hook /etc/tklbam/hooks.d/example +contrib/example-hook-pre-backup /etc/tklbam/hooks.d/example-pre-backup +contrib/example-hook-post-restore /etc/tklbam/hooks.d/example-post-restore +contrib/ez-apt-install.sh /usr/share/tklbam/contrib/ez-apt-install.sh -contrib/example-hook => /etc/tklbam/hooks.d/example -contrib/example-hook-pre-backup => /etc/tklbam/hooks.d/example-pre-backup -contrib/example-hook-post-restore => /etc/tklbam/hooks.d/example-post-restore -contrib/ez-apt-install.sh => /usr/share/tklbam/contrib/ez-apt-install.sh - -deproot => /usr/lib/tklbam/deps +lib/* /usr/lib/tklbam/ diff --git a/debian/tklbam.links b/debian/tklbam.links new file mode 100644 index 0000000..457b262 --- /dev/null +++ b/debian/tklbam.links @@ -0,0 +1,10 @@ +usr/bin/tklbam /usr/lib/tklbam/cmd.py +usr/bin/tklbam-backup /usr/lib/tklbam/cmd_backup.py +usr/bin/tklbam-escrow /usr/lib/tklbam/cmd_escrow.py +usr/bin/tklbam-init /usr/lib/tklbam/cmd_init.py +usr/bin/tklbam-internal /usr/lib/tklbam/cmd_internal.py +usr/bin/tklbam-list /usr/lib/tklbam/cmd_list.py +usr/bin/tklbam-passphrase /usr/lib/tklbam/cmd_passphrase.py +usr/bin/tklbam-restore /usr/lib/tklbam/cmd_restore.py +usr/bin/tklbam-restore-rollback /usr/lib/tklbam/cmd_restore_rollback.py +usr/bin/tklbam-status /usr/lib/tklbam/cmd_status.py From 57278c948d9a653ce134c62fbae9839bdfdf9b43 Mon Sep 17 00:00:00 2001 From: Jeremy Davis Date: Sun, 21 Sep 2025 23:05:44 +0000 Subject: [PATCH 4/9] support arm64 too, add update_pypy.sh helper & commit checksums.txt --- checksums.txt | 3 +++ debian/control | 4 +-- duct-tape-deps.sh | 63 ++++++++++++++++++++++++++++------------------- update_pypy.sh | 28 +++++++++++++++++++++ 4 files changed, 70 insertions(+), 28 deletions(-) create mode 100644 checksums.txt create mode 100755 update_pypy.sh diff --git a/checksums.txt b/checksums.txt new file mode 100644 index 0000000..57a7e7c --- /dev/null +++ b/checksums.txt @@ -0,0 +1,3 @@ +# pypy versions and checksums - run ./update_pypy.sh to update +aa3bb92dbb529fa2d4920895b16d67a810b0c709207857d56cfe4a6e3b41e02a pypy2.7-v7.3.20-linux64.tar.bz2 +f22a1be607deeaa4f9be6bc63aae09fe4fb5b990d6a23aa4e7c5960dc5d93c96 pypy2.7-v7.3.20-aarch64.tar.bz2 diff --git a/debian/control b/debian/control index 68b94e7..5eadde2 100644 --- a/debian/control +++ b/debian/control @@ -8,11 +8,11 @@ Build-Depends: libcurl4-openssl-dev, wget, tar, - libssl-dev + libssl-dev, Standards-Version: 4.0.0 Package: tklbam -Architecture: all +Architecture: any Depends: ca-certificates, ${misc:Depends}, diff --git a/duct-tape-deps.sh b/duct-tape-deps.sh index 8f5618d..3cfc0f3 100755 --- a/duct-tape-deps.sh +++ b/duct-tape-deps.sh @@ -1,34 +1,45 @@ -#!/bin/bash +#!/bin/bash -eu -pypy_root='pypy2.7-v7.3.20-linux64' -pypy_archive="$pypy_root.tar.bz2" +BASE_DIR="$PWD" +DEPROOT="$BASE_DIR/lib/deps" +TMP="$BASE_DIR/debian/tmp/tklbam-deps" +HOST_ARCH=$(dpkg --print-architecture) -DEPROOT="$(pwd)/debian/tmp/dh-*/deproot" +export LD_LIBRARY_PATH="$DEPROOT/bin" -mkdir -p $DEPROOT /usr/lib/tklbam/deps +mkdir -p "$DEPROOT" "$TMP" -cd debian/tmp +pypy_arch= +case $HOST_ARCH in + amd64) + pypy_arch="linux64";; + arm64) + pypy_arch="aarch64";; + *) + echo "ERROR: $HOST_ARCH unsupported" >&2 + exit 1;; +esac -wget "https://downloads.python.org/pypy/$pypy_archive" -tar -xvf "$pypy_archive" - -pypy_root="$(pwd)/$pypy_root" +read -r pypy_checksum pypy_archive <<< "$( \ + sed -n "/pypy2\.7.*$pypy_arch/{s|<*.*>||;p;q;}" "checksums.txt" \ +)" -rm "$pypy_archive" +cd "$TMP" || exit 1 -git clone https://github.com/turnkeylinux/python-crypto -cd python-crypto -LD_LIBRARY_PATH="$LD_LIBRARY_PATH:$pypy_root/bin" "$pypy_root/bin/python2" setup.py build - -cd .. -git clone https://github.com/turnkeylinux/python-pycurl -cd python-pycurl -LD_LIBRARY_PATH="$LD_LIBRARY_PATH:$pypy_root/bin" "$pypy_root/bin/python2" setup.py build +wget "https://downloads.python.org/pypy/$pypy_archive" -cd .. -mv pypy*/* $DEPROOT -mv python-crypto/build/lib*/* $DEPROOT/site-packages -rm -rf python-crypto -mv python-pycurl/build/lib*/* $DEPROOT/site-packages -rm -rf python-pycurl -cd .. +if [[ $(sha256sum "$pypy_archive") != "$pypy_checksum"*"$pypy_archive" ]]; then + echo "ERROR: $pypy_archive checksum mismatch" >&2 + exit 1 +fi + +echo "unpacking $pypy_archive..." +tar -xf "$pypy_archive" --transform "s|^${pypy_archive%.tar.bz2}/||" -C "$DEPROOT" + +for pkg in python-crypto python-pycurl; do + git clone https://github.com/turnkeylinux/$pkg + cd $pkg || exit 1 + "$LD_LIBRARY_PATH/pypy" setup.py build + cd "$TMP" || exit 1 + mv $pkg/build/lib*/* $DEPROOT/site-packages +done diff --git a/update_pypy.sh b/update_pypy.sh new file mode 100755 index 0000000..7c90c0f --- /dev/null +++ b/update_pypy.sh @@ -0,0 +1,28 @@ +#!/bin/bash -eu + +# Run this script to update the version of pypy to be dynamically downloaded +# when building the tklbam deb package. Will always download the latest pypy. + +# pypy arch -> debian arch: +# - linux64 = amd64 +# - aarch64 = arm64 + +SUPPORTED_ARCH=(linux64 aarch64) + +tmp_file=$(mktemp) + +curl --silent https://pypy.org/checksums.html --output "$tmp_file" + +echo "# pypy versions and checksums - run ./update_pypy.sh to update" > \ + checksums.txt +for arch in "${SUPPORTED_ARCH[@]}"; do + sed -n "/pypy2\.7.*$arch/{s|<*.*>||;p;q;}" "$tmp_file" >> checksums.txt +done +rm -rf "$tmp_file" +if git diff --quiet HEAD -- checksums.txt; then + echo "no update available - checksums.txt unchanged" +else + echo "checksums.txt updated; double check and commit" + # non-zero exit if there's something to do + exit 1 +fi From d3ea69f841e369fc6ff3f31ae86a16f19881fc4d Mon Sep 17 00:00:00 2001 From: Stefan Davis Date: Tue, 23 Sep 2025 10:35:27 +1000 Subject: [PATCH 5/9] update path & shebang --- lib/cmd.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/lib/cmd.py b/lib/cmd.py index 5c9e157..bd31af2 100755 --- a/lib/cmd.py +++ b/lib/cmd.py @@ -1,4 +1,4 @@ -#!/usr/bin/python2 +#!/usr/lib/tklbam/deps/bin/python2 # # Copyright (c) 2010-2012 Liraz Siri # @@ -21,6 +21,13 @@ Default: $TKLBAM_REGISTRY """ +import sys +sys.path = [ + '', + '/usr/lib/tklbam/deps/lib_pypy', + '/usr/lib/tklbam/deps/lib-python/2.7', + '/usr/lib/tklbam/deps/site-packages', +] from os.path import * from cliwrapper import CliWrapper From 25540416f349ec687567a070b0daf92f0fcc6b16 Mon Sep 17 00:00:00 2001 From: Stefan Davis Date: Wed, 24 Sep 2025 11:16:26 +1000 Subject: [PATCH 6/9] fix links --- debian/tklbam.links | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/debian/tklbam.links b/debian/tklbam.links index 457b262..baeba1b 100644 --- a/debian/tklbam.links +++ b/debian/tklbam.links @@ -1,10 +1,10 @@ -usr/bin/tklbam /usr/lib/tklbam/cmd.py -usr/bin/tklbam-backup /usr/lib/tklbam/cmd_backup.py -usr/bin/tklbam-escrow /usr/lib/tklbam/cmd_escrow.py -usr/bin/tklbam-init /usr/lib/tklbam/cmd_init.py -usr/bin/tklbam-internal /usr/lib/tklbam/cmd_internal.py -usr/bin/tklbam-list /usr/lib/tklbam/cmd_list.py -usr/bin/tklbam-passphrase /usr/lib/tklbam/cmd_passphrase.py -usr/bin/tklbam-restore /usr/lib/tklbam/cmd_restore.py -usr/bin/tklbam-restore-rollback /usr/lib/tklbam/cmd_restore_rollback.py -usr/bin/tklbam-status /usr/lib/tklbam/cmd_status.py +/usr/lib/tklbam/cmd.py /usr/bin/tklbam +/usr/lib/tklbam/cmd_backup.py /usr/bin/tklbam-backup +/usr/lib/tklbam/cmd_escrow.py /usr/bin/tklbam-escrow +/usr/lib/tklbam/cmd_init.py /usr/bin/tklbam-init +/usr/lib/tklbam/cmd_internal.py /usr/bin/tklbam-internal +/usr/lib/tklbam/cmd_list.py /usr/bin/tklbam-list +/usr/lib/tklbam/cmd_passphrase.py /usr/bin/tklbam-passphrase +/usr/lib/tklbam/cmd_restore.py /usr/bin/tklbam-restore +/usr/lib/tklbam/cmd_restore_rollback.py /usr/bin/tklbam-restore-rollback +/usr/lib/tklbam/cmd_status.py /usr/bin/tklbam-status From c703bdae8d5d6451931402abbe84e221bcc7de3a Mon Sep 17 00:00:00 2001 From: Stefan Davis Date: Wed, 24 Sep 2025 11:56:32 +1000 Subject: [PATCH 7/9] embed more deps --- debian/control | 9 ++------- duct-tape-deps.sh | 9 ++++++++- lib/backup.py | 6 +++--- lib/cmd.py | 1 + lib/registry.py | 6 +++--- lib/restore.py | 4 ++-- 6 files changed, 19 insertions(+), 16 deletions(-) diff --git a/debian/control b/debian/control index 5eadde2..8bec6c5 100644 --- a/debian/control +++ b/debian/control @@ -8,6 +8,7 @@ Build-Depends: libcurl4-openssl-dev, wget, tar, + librsync-dev (>= 0.9.6), libssl-dev, Standards-Version: 4.0.0 @@ -18,12 +19,6 @@ Depends: ${misc:Depends}, gnupg, ntpdate, - pycurl-wrapper (>= 1.2), - pycurl-wrapper (<< 2.0), gpg (>= 2.2.12), - python-simplejson, - tklbam-duplicity (>= 0.6.18), - tklbam-python-boto (>= 2.3.0-2turnkey), - tklbam-squid (>= 2.7.STABLE9-2.1turnkey+25), - turnkey-pylib (>= 0.5), + tklbam-squid Description: TurnKey GNU/Linux Backup and Migration agent diff --git a/duct-tape-deps.sh b/duct-tape-deps.sh index 3cfc0f3..230dc54 100755 --- a/duct-tape-deps.sh +++ b/duct-tape-deps.sh @@ -36,10 +36,17 @@ fi echo "unpacking $pypy_archive..." tar -xf "$pypy_archive" --transform "s|^${pypy_archive%.tar.bz2}/||" -C "$DEPROOT" -for pkg in python-crypto python-pycurl; do +for pkg in python-crypto python-pycurl tklbam-duplicity turnkey-pylib tklbam-python-boto; do git clone https://github.com/turnkeylinux/$pkg cd $pkg || exit 1 "$LD_LIBRARY_PATH/pypy" setup.py build cd "$TMP" || exit 1 mv $pkg/build/lib*/* $DEPROOT/site-packages done + +pkg=pycurl-wrapper +git clone -b python2 https://github.com/turnkeylinux/$pkg +cd $pkg || exit 1 +"$LD_LIBRARY_PATH/pypy" setup.py build +cd "$TMP" || exit 1 +mv $pkg/build/lib*/* $DEPROOT/site-packages diff --git a/lib/backup.py b/lib/backup.py index 4b39799..81d0137 100644 --- a/lib/backup.py +++ b/lib/backup.py @@ -15,7 +15,7 @@ import stat import shutil -import simplejson +import json from paths import Paths @@ -73,12 +73,12 @@ def fromfile(cls, path): if not exists(path): return None - d = simplejson.load(file(path)) + d = json.load(file(path)) return cls(*(d[attr] for attr in ('profile_id', 'overrides', 'skip_files', 'skip_packages', 'skip_database'))) def tofile(self, path): - simplejson.dump(dict(self), file(path, "w")) + json.dump(dict(self), file(path, "w")) class Backup: class Error(Exception): diff --git a/lib/cmd.py b/lib/cmd.py index bd31af2..8088102 100755 --- a/lib/cmd.py +++ b/lib/cmd.py @@ -24,6 +24,7 @@ import sys sys.path = [ '', + '/usr/lib/tklbam', '/usr/lib/tklbam/deps/lib_pypy', '/usr/lib/tklbam/deps/lib-python/2.7', '/usr/lib/tklbam/deps/site-packages', diff --git a/lib/registry.py b/lib/registry.py index 1ac7c2d..d6582c6 100644 --- a/lib/registry.py +++ b/lib/registry.py @@ -15,7 +15,7 @@ import re from os.path import * from paths import Paths as _Paths -import simplejson +import json from datetime import datetime @@ -204,12 +204,12 @@ def backup_resume_conf(self, val=UNDEFINED): return try: - return BackupSessionConf(simplejson.loads(s)) + return BackupSessionConf(json.loads(s)) except: return else: - s = simplejson.dumps(val) + s = json.dumps(val) self._file_str(self.path.backup_resume, s) backup_resume_conf = property(backup_resume_conf, backup_resume_conf) diff --git a/lib/restore.py b/lib/restore.py index ed7469b..d1556d6 100644 --- a/lib/restore.py +++ b/lib/restore.py @@ -27,7 +27,7 @@ import mysql import pgsql -import simplejson +import json from temp import TempFile @@ -52,7 +52,7 @@ def __init__(self, backup_extract_path, limits=[], rollback=True, simulate=False if simulate: rollback = False - self.conf = AttrDict(simplejson.loads(file(self.extras.backup_conf).read())) \ + self.conf = AttrDict(json.loads(file(self.extras.backup_conf).read())) \ if exists(self.extras.backup_conf) else None self.simulate = simulate From e3c3e6817f662cff65176ff23a410be1fc889d78 Mon Sep 17 00:00:00 2001 From: Stefan Davis Date: Mon, 29 Sep 2025 12:32:12 +1000 Subject: [PATCH 8/9] fix install file --- debian/tklbam.install | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) mode change 100644 => 100755 debian/tklbam.install diff --git a/debian/tklbam.install b/debian/tklbam.install old mode 100644 new mode 100755 index 1de7eb1..24e4549 --- a/debian/tklbam.install +++ b/debian/tklbam.install @@ -1,12 +1,17 @@ -contrib/example-conf /etc/tklbam/conf -contrib/example-overrides /etc/tklbam/overrides -contrib/fixclock-hook /etc/tklbam/hooks.d/fixclock -contrib/maria-db-changes-hook /etc/tklbam/hooks.d/maria-db-changes -contrib/cron.sh /etc/cron.daily/tklbam-backup +#!/usr/bin/dh-exec -contrib/example-hook /etc/tklbam/hooks.d/example -contrib/example-hook-pre-backup /etc/tklbam/hooks.d/example-pre-backup -contrib/example-hook-post-restore /etc/tklbam/hooks.d/example-post-restore -contrib/ez-apt-install.sh /usr/share/tklbam/contrib/ez-apt-install.sh +# rename files +contrib/example-conf => /etc/tklbam/conf +contrib/example-overrides => /etc/tklbam/overrides +contrib/fixclock-hook => /etc/tklbam/hooks.d/fixclock +contrib/maria-db-changes-hook => /etc/tklbam/hooks.d/maria-db-changes +contrib/cron.sh => /etc/cron.daily/tklbam-backup +contrib/example-hook => /etc/tklbam/hooks.d/example +contrib/example-hook-pre-backup => /etc/tklbam/hooks.d/example-pre-backup +contrib/example-hook-post-restore => /etc/tklbam/hooks.d/example-post-restore +contrib/ez-apt-install.sh => /usr/share/tklbam/contrib/ez-apt-install.sh + +# move files/dirs lib/* /usr/lib/tklbam/ + From 36f0386a49ce273c1eb92f425e4ab98c9f21c4a6 Mon Sep 17 00:00:00 2001 From: Stefan Davis Date: Fri, 3 Oct 2025 10:51:08 +1000 Subject: [PATCH 9/9] get each dep from specific commits --- dep-commit-ids | 6 ++++++ duct-tape-deps.sh | 21 +++++++++------------ update-dep-versions.sh | 21 +++++++++++++++++++++ 3 files changed, 36 insertions(+), 12 deletions(-) create mode 100644 dep-commit-ids create mode 100755 update-dep-versions.sh diff --git a/dep-commit-ids b/dep-commit-ids new file mode 100644 index 0000000..d73f622 --- /dev/null +++ b/dep-commit-ids @@ -0,0 +1,6 @@ +python-crypto:432d70d8c4120643955e877f59522c8b3f248bc7 +python-pycurl:44b7871c4daeb6d3298a7be11840c9821798c56a +tklbam-duplicity:c5f4d9a09c4c127ed38720ed1848c04d40bc4612 +turnkey-pylib:accf6fb9a26cee6b20c42606498bafef2b5a77cf +tklbam-python-boto:aa3759b09b3a81831270d3627d11a8433f7ebf32 +pycurl-wrapper:adde46c337131f01d9f0e96259625e92ccfa37fa diff --git a/duct-tape-deps.sh b/duct-tape-deps.sh index 230dc54..e1e2452 100755 --- a/duct-tape-deps.sh +++ b/duct-tape-deps.sh @@ -36,17 +36,14 @@ fi echo "unpacking $pypy_archive..." tar -xf "$pypy_archive" --transform "s|^${pypy_archive%.tar.bz2}/||" -C "$DEPROOT" -for pkg in python-crypto python-pycurl tklbam-duplicity turnkey-pylib tklbam-python-boto; do - git clone https://github.com/turnkeylinux/$pkg - cd $pkg || exit 1 +while IFS= read -r line; do + pkg="${line%:*}" + commit_id="${line##*:}" + + git clone "https://github.com/turnkeylinux/$pkg" + cd "$pkg" || exit 1 + git checkout "$commit_id" "$LD_LIBRARY_PATH/pypy" setup.py build cd "$TMP" || exit 1 - mv $pkg/build/lib*/* $DEPROOT/site-packages -done - -pkg=pycurl-wrapper -git clone -b python2 https://github.com/turnkeylinux/$pkg -cd $pkg || exit 1 -"$LD_LIBRARY_PATH/pypy" setup.py build -cd "$TMP" || exit 1 -mv $pkg/build/lib*/* $DEPROOT/site-packages + mv "$pkg/build/lib"*/* "$DEPROOT/site-packages" +done < "$BASE_DIR/dep-commit-ids" diff --git a/update-dep-versions.sh b/update-dep-versions.sh new file mode 100755 index 0000000..b596ea3 --- /dev/null +++ b/update-dep-versions.sh @@ -0,0 +1,21 @@ +#!/bin/bash -eu + + +tmp_file="$(mktemp -d)" + +COMMIT_ID_PATH="$(pwd)/dep-commit-ids" + +rm -f "$COMMIT_ID_PATH" + +cd "$tmp_file" +for pkg in python-crypto python-pycurl tklbam-duplicity turnkey-pylib tklbam-python-boto pycurl-wrapper; do + echo "# getting HEAD commit id for $pkg" + if [[ $pkg == "pycurl-wrapper" ]]; then + commit_id="$(git ls-remote "https://github.com/turnkeylinux/$pkg" | grep refs/heads/python2 | awk '{ print $1 }')" + else + commit_id="$(git ls-remote "https://github.com/turnkeylinux/$pkg" | grep HEAD | awk '{ print $1 }')" + fi + echo "$pkg:$commit_id" >> "$COMMIT_ID_PATH" +done + +rm -r "$tmp_file"