diff --git a/Makefile b/Makefile deleted file mode 100644 index b8f27a7..0000000 --- a/Makefile +++ /dev/null @@ -1,96 +0,0 @@ -# standard Python project Makefile -progname = $(shell awk '/^Source/ {print $$2}' debian/control) -name= - -prefix = /usr/local -PATH_BIN = $(prefix)/bin -PATH_INSTALL_LIB = $(prefix)/lib/$(progname) -PATH_DIST := $(progname)-$$(autoversion HEAD) - -SHELL = /bin/bash - -all: help - -debug: - $(foreach v, $V, $(warning $v = $($v))) - @true - -dist: - -mkdir -p $(PATH_DIST) - - cp -a .git $(PATH_DIST) - cd $(PATH_DIST) && git-checkout --force HEAD - - tar jcvf $(PATH_DIST).tar.bz2 $(PATH_DIST) - rm -rf $(PATH_DIST) - -#docs: -# $(MAKE) -C docs - -help: - @echo '=== Targets:' - @echo 'install [ prefix=path/to/usr ] # default: prefix=$(value prefix)' - @echo 'uninstall [ prefix=path/to/usr ]' - @echo 'docs' - @echo - @echo 'clean' - -# DRY macros -debbuild=debian/$(shell awk '/^Package/ {print $$2}' debian/control) - -truepath = $(shell echo $1 | sed -e 's|^$(debbuild)||') -libpath = $(call truepath,$(PATH_INSTALL_LIB))/$$(basename $1) -subcommand = $(progname)-$$(echo $1 | sed 's|.*/||; s/^cmd_//; s/_/-/g; s/.py$$//') -echo-do = echo $1; $1 - -# first argument: code we execute if there is just one executable module -# second argument: code we execute if there is more than on executable module -define with-py-executables - @modules=$$(find -maxdepth 1 -type f -name 'cmd_*.py' -perm -100); \ - modules_len=$$(echo $$modules | wc -w); \ - if [ $$modules_len = 1 ]; then \ - module=$$modules; \ - $(call echo-do, $1); \ - elif [ $$modules_len -gt 1 ]; then \ - for module in $$modules; do \ - $(call echo-do, $2); \ - done; \ - fi; -endef - -install: #docs - @echo - @echo \*\* CONFIG: prefix = $(prefix) \*\* - @echo - - install -d $(PATH_BIN) $(PATH_INSTALL_LIB) - cp *.py $(PATH_INSTALL_LIB) - - install -d $(PATH_INSTALL_LIB)/cmd_internals - cp cmd_internals/*.py $(PATH_INSTALL_LIB)/cmd_internals - - $(call with-py-executables, \ - ln -fs $(call libpath, $$module) $(PATH_BIN)/$(progname), \ - ln -fs $(call libpath, $$module) $(PATH_BIN)/$(call subcommand, $$module)) - - ln -fs $(call libpath, cmd.py) $(PATH_BIN)/$(progname) - -uninstall: - rm -rf $(PATH_INSTALL_LIB) - - $(call with-py-executables, \ - rm -f $(PATH_BIN)/$(progname), \ - rm -f $(PATH_BIN)/$(call subcommand, $$module)) - -clean: - $(MAKE) -C docs clean - - rm -f {,*/}*.pyc {,*/}*.pyo - - @emptydirs=$$(find -type d -empty); \ - if [ "$$emptydirs" ]; then \ - echo rmdir $$emptydirs; \ - rmdir $$emptydirs; \ - fi - -.PHONY: all debug dist docs help install uninstall clean diff --git a/checksums.txt b/checksums.txt new file mode 100644 index 0000000..57a7e7c --- /dev/null +++ b/checksums.txt @@ -0,0 +1,3 @@ +# pypy versions and checksums - run ./update_pypy.sh to update +aa3bb92dbb529fa2d4920895b16d67a810b0c709207857d56cfe4a6e3b41e02a pypy2.7-v7.3.20-linux64.tar.bz2 +f22a1be607deeaa4f9be6bc63aae09fe4fb5b990d6a23aa4e7c5960dc5d93c96 pypy2.7-v7.3.20-aarch64.tar.bz2 diff --git a/debian/control b/debian/control index df6ebf9..8bec6c5 100644 --- a/debian/control +++ b/debian/control @@ -5,26 +5,20 @@ Maintainer: Liraz Siri Build-Depends: debhelper (>= 10), dh-exec, - dh-python2 (>= 4.20201102+turnkey0), - python-all (>= 2.6.6-3~), + libcurl4-openssl-dev, + wget, + tar, + librsync-dev (>= 0.9.6), + libssl-dev, Standards-Version: 4.0.0 -X-Python-Version: >= 2.6 Package: tklbam -Architecture: all +Architecture: any Depends: ca-certificates, ${misc:Depends}, gnupg, ntpdate, - pycurl-wrapper (>= 1.2), - pycurl-wrapper (<< 2.0), gpg (>= 2.2.12), - ${python:Depends}, - python-crypto, - python-simplejson, - tklbam-duplicity (>= 0.6.18), - tklbam-python-boto (>= 2.3.0-2turnkey), - tklbam-squid (>= 2.7.STABLE9-2.1turnkey+25), - turnkey-pylib (>= 0.5), + tklbam-squid Description: TurnKey GNU/Linux Backup and Migration agent diff --git a/debian/rules b/debian/rules index 500f8be..50c48b1 100755 --- a/debian/rules +++ b/debian/rules @@ -1,9 +1,16 @@ #! /usr/bin/make -f -include /usr/share/dpkg/pkg-info.mk +progname=$(shell awk '/^Source/ {print $$2}' debian/control) +buildroot=debian/$(progname) + +deproot=$(progname)/deproot +tmp=debian/tmp %: - dh $@ --with python2 --buildsystem=makefile + dh $@ + +override_dh_auto_build: + ./duct-tape-deps.sh -override_dh_auto_install: - dh_auto_install -- prefix=debian/$(DEB_SOURCE)/usr +override_dh_shlibdeps: +override_dh_makeshlibs: diff --git a/debian/tklbam.install b/debian/tklbam.install index 28a5861..24e4549 100755 --- a/debian/tklbam.install +++ b/debian/tklbam.install @@ -1,12 +1,17 @@ #!/usr/bin/dh-exec -contrib/example-conf => /etc/tklbam/conf -contrib/example-overrides => /etc/tklbam/overrides -contrib/fixclock-hook => /etc/tklbam/hooks.d/fixclock -contrib/maria-db-changes-hook => /etc/tklbam/hooks.d/maria-db-changes -contrib/cron.sh => /etc/cron.daily/tklbam-backup - -contrib/example-hook => /etc/tklbam/hooks.d/example -contrib/example-hook-pre-backup => /etc/tklbam/hooks.d/example-pre-backup -contrib/example-hook-post-restore => /etc/tklbam/hooks.d/example-post-restore -contrib/ez-apt-install.sh => /usr/share/tklbam/contrib/ez-apt-install.sh +# rename files +contrib/example-conf => /etc/tklbam/conf +contrib/example-overrides => /etc/tklbam/overrides +contrib/fixclock-hook => /etc/tklbam/hooks.d/fixclock +contrib/maria-db-changes-hook => /etc/tklbam/hooks.d/maria-db-changes +contrib/cron.sh => /etc/cron.daily/tklbam-backup + +contrib/example-hook => /etc/tklbam/hooks.d/example +contrib/example-hook-pre-backup => /etc/tklbam/hooks.d/example-pre-backup +contrib/example-hook-post-restore => /etc/tklbam/hooks.d/example-post-restore +contrib/ez-apt-install.sh => /usr/share/tklbam/contrib/ez-apt-install.sh + +# move files/dirs +lib/* /usr/lib/tklbam/ + diff --git a/debian/tklbam.links b/debian/tklbam.links new file mode 100644 index 0000000..baeba1b --- /dev/null +++ b/debian/tklbam.links @@ -0,0 +1,10 @@ +/usr/lib/tklbam/cmd.py /usr/bin/tklbam +/usr/lib/tklbam/cmd_backup.py /usr/bin/tklbam-backup +/usr/lib/tklbam/cmd_escrow.py /usr/bin/tklbam-escrow +/usr/lib/tklbam/cmd_init.py /usr/bin/tklbam-init +/usr/lib/tklbam/cmd_internal.py /usr/bin/tklbam-internal +/usr/lib/tklbam/cmd_list.py /usr/bin/tklbam-list +/usr/lib/tklbam/cmd_passphrase.py /usr/bin/tklbam-passphrase +/usr/lib/tklbam/cmd_restore.py /usr/bin/tklbam-restore +/usr/lib/tklbam/cmd_restore_rollback.py /usr/bin/tklbam-restore-rollback +/usr/lib/tklbam/cmd_status.py /usr/bin/tklbam-status diff --git a/dep-commit-ids b/dep-commit-ids new file mode 100644 index 0000000..d73f622 --- /dev/null +++ b/dep-commit-ids @@ -0,0 +1,6 @@ +python-crypto:432d70d8c4120643955e877f59522c8b3f248bc7 +python-pycurl:44b7871c4daeb6d3298a7be11840c9821798c56a +tklbam-duplicity:c5f4d9a09c4c127ed38720ed1848c04d40bc4612 +turnkey-pylib:accf6fb9a26cee6b20c42606498bafef2b5a77cf +tklbam-python-boto:aa3759b09b3a81831270d3627d11a8433f7ebf32 +pycurl-wrapper:adde46c337131f01d9f0e96259625e92ccfa37fa diff --git a/duct-tape-deps.sh b/duct-tape-deps.sh new file mode 100755 index 0000000..e1e2452 --- /dev/null +++ b/duct-tape-deps.sh @@ -0,0 +1,49 @@ +#!/bin/bash -eu + +BASE_DIR="$PWD" +DEPROOT="$BASE_DIR/lib/deps" +TMP="$BASE_DIR/debian/tmp/tklbam-deps" +HOST_ARCH=$(dpkg --print-architecture) + +export LD_LIBRARY_PATH="$DEPROOT/bin" + +mkdir -p "$DEPROOT" "$TMP" + +pypy_arch= +case $HOST_ARCH in + amd64) + pypy_arch="linux64";; + arm64) + pypy_arch="aarch64";; + *) + echo "ERROR: $HOST_ARCH unsupported" >&2 + exit 1;; +esac + +read -r pypy_checksum pypy_archive <<< "$( \ + sed -n "/pypy2\.7.*$pypy_arch/{s|<*.*>||;p;q;}" "checksums.txt" \ +)" + +cd "$TMP" || exit 1 + +wget "https://downloads.python.org/pypy/$pypy_archive" + +if [[ $(sha256sum "$pypy_archive") != "$pypy_checksum"*"$pypy_archive" ]]; then + echo "ERROR: $pypy_archive checksum mismatch" >&2 + exit 1 +fi + +echo "unpacking $pypy_archive..." +tar -xf "$pypy_archive" --transform "s|^${pypy_archive%.tar.bz2}/||" -C "$DEPROOT" + +while IFS= read -r line; do + pkg="${line%:*}" + commit_id="${line##*:}" + + git clone "https://github.com/turnkeylinux/$pkg" + cd "$pkg" || exit 1 + git checkout "$commit_id" + "$LD_LIBRARY_PATH/pypy" setup.py build + cd "$TMP" || exit 1 + mv "$pkg/build/lib"*/* "$DEPROOT/site-packages" +done < "$BASE_DIR/dep-commit-ids" diff --git a/backup.py b/lib/backup.py similarity index 98% rename from backup.py rename to lib/backup.py index 4b39799..81d0137 100644 --- a/backup.py +++ b/lib/backup.py @@ -15,7 +15,7 @@ import stat import shutil -import simplejson +import json from paths import Paths @@ -73,12 +73,12 @@ def fromfile(cls, path): if not exists(path): return None - d = simplejson.load(file(path)) + d = json.load(file(path)) return cls(*(d[attr] for attr in ('profile_id', 'overrides', 'skip_files', 'skip_packages', 'skip_database'))) def tofile(self, path): - simplejson.dump(dict(self), file(path, "w")) + json.dump(dict(self), file(path, "w")) class Backup: class Error(Exception): diff --git a/changes.py b/lib/changes.py similarity index 100% rename from changes.py rename to lib/changes.py diff --git a/cliwrapper.py b/lib/cliwrapper.py similarity index 100% rename from cliwrapper.py rename to lib/cliwrapper.py diff --git a/cmd.py b/lib/cmd.py similarity index 86% rename from cmd.py rename to lib/cmd.py index 5c9e157..8088102 100755 --- a/cmd.py +++ b/lib/cmd.py @@ -1,4 +1,4 @@ -#!/usr/bin/python2 +#!/usr/lib/tklbam/deps/bin/python2 # # Copyright (c) 2010-2012 Liraz Siri # @@ -21,6 +21,14 @@ Default: $TKLBAM_REGISTRY """ +import sys +sys.path = [ + '', + '/usr/lib/tklbam', + '/usr/lib/tklbam/deps/lib_pypy', + '/usr/lib/tklbam/deps/lib-python/2.7', + '/usr/lib/tklbam/deps/site-packages', +] from os.path import * from cliwrapper import CliWrapper diff --git a/cmd_backup.py b/lib/cmd_backup.py similarity index 100% rename from cmd_backup.py rename to lib/cmd_backup.py diff --git a/cmd_escrow.py b/lib/cmd_escrow.py similarity index 100% rename from cmd_escrow.py rename to lib/cmd_escrow.py diff --git a/cmd_init.py b/lib/cmd_init.py similarity index 100% rename from cmd_init.py rename to lib/cmd_init.py diff --git a/cmd_internal.py b/lib/cmd_internal.py similarity index 100% rename from cmd_internal.py rename to lib/cmd_internal.py diff --git a/cmd_internals/__init__.py b/lib/cmd_internals/__init__.py similarity index 100% rename from cmd_internals/__init__.py rename to lib/cmd_internals/__init__.py diff --git a/cmd_internals/cmd_create_profile.py b/lib/cmd_internals/cmd_create_profile.py similarity index 100% rename from cmd_internals/cmd_create_profile.py rename to lib/cmd_internals/cmd_create_profile.py diff --git a/cmd_internals/cmd_delete.py b/lib/cmd_internals/cmd_delete.py similarity index 100% rename from cmd_internals/cmd_delete.py rename to lib/cmd_internals/cmd_delete.py diff --git a/cmd_internals/cmd_detect_profile_id.py b/lib/cmd_internals/cmd_detect_profile_id.py similarity index 100% rename from cmd_internals/cmd_detect_profile_id.py rename to lib/cmd_internals/cmd_detect_profile_id.py diff --git a/cmd_internals/cmd_dirindex.py b/lib/cmd_internals/cmd_dirindex.py similarity index 100% rename from cmd_internals/cmd_dirindex.py rename to lib/cmd_internals/cmd_dirindex.py diff --git a/cmd_internals/cmd_fixstat.py b/lib/cmd_internals/cmd_fixstat.py similarity index 100% rename from cmd_internals/cmd_fixstat.py rename to lib/cmd_internals/cmd_fixstat.py diff --git a/cmd_internals/cmd_fs2mysql.py b/lib/cmd_internals/cmd_fs2mysql.py similarity index 100% rename from cmd_internals/cmd_fs2mysql.py rename to lib/cmd_internals/cmd_fs2mysql.py diff --git a/cmd_internals/cmd_fs2pgsql.py b/lib/cmd_internals/cmd_fs2pgsql.py similarity index 100% rename from cmd_internals/cmd_fs2pgsql.py rename to lib/cmd_internals/cmd_fs2pgsql.py diff --git a/cmd_internals/cmd_merge_userdb.py b/lib/cmd_internals/cmd_merge_userdb.py similarity index 100% rename from cmd_internals/cmd_merge_userdb.py rename to lib/cmd_internals/cmd_merge_userdb.py diff --git a/cmd_internals/cmd_mysql2fs.py b/lib/cmd_internals/cmd_mysql2fs.py similarity index 100% rename from cmd_internals/cmd_mysql2fs.py rename to lib/cmd_internals/cmd_mysql2fs.py diff --git a/cmd_internals/cmd_newpkgs.py b/lib/cmd_internals/cmd_newpkgs.py similarity index 100% rename from cmd_internals/cmd_newpkgs.py rename to lib/cmd_internals/cmd_newpkgs.py diff --git a/cmd_internals/cmd_newpkgs_install.py b/lib/cmd_internals/cmd_newpkgs_install.py similarity index 100% rename from cmd_internals/cmd_newpkgs_install.py rename to lib/cmd_internals/cmd_newpkgs_install.py diff --git a/cmd_internals/cmd_pgsql2fs.py b/lib/cmd_internals/cmd_pgsql2fs.py similarity index 100% rename from cmd_internals/cmd_pgsql2fs.py rename to lib/cmd_internals/cmd_pgsql2fs.py diff --git a/cmd_internals/cmd_stsagent.py b/lib/cmd_internals/cmd_stsagent.py similarity index 100% rename from cmd_internals/cmd_stsagent.py rename to lib/cmd_internals/cmd_stsagent.py diff --git a/cmd_list.py b/lib/cmd_list.py similarity index 100% rename from cmd_list.py rename to lib/cmd_list.py diff --git a/cmd_passphrase.py b/lib/cmd_passphrase.py similarity index 100% rename from cmd_passphrase.py rename to lib/cmd_passphrase.py diff --git a/cmd_restore.py b/lib/cmd_restore.py similarity index 100% rename from cmd_restore.py rename to lib/cmd_restore.py diff --git a/cmd_restore_rollback.py b/lib/cmd_restore_rollback.py similarity index 100% rename from cmd_restore_rollback.py rename to lib/cmd_restore_rollback.py diff --git a/cmd_status.py b/lib/cmd_status.py similarity index 100% rename from cmd_status.py rename to lib/cmd_status.py diff --git a/conf.py b/lib/conf.py similarity index 100% rename from conf.py rename to lib/conf.py diff --git a/dblimits.py b/lib/dblimits.py similarity index 100% rename from dblimits.py rename to lib/dblimits.py diff --git a/dirindex.py b/lib/dirindex.py similarity index 100% rename from dirindex.py rename to lib/dirindex.py diff --git a/dummyhub.py b/lib/dummyhub.py similarity index 100% rename from dummyhub.py rename to lib/dummyhub.py diff --git a/duplicity.py b/lib/duplicity.py similarity index 100% rename from duplicity.py rename to lib/duplicity.py diff --git a/hooks.py b/lib/hooks.py similarity index 100% rename from hooks.py rename to lib/hooks.py diff --git a/hub.py b/lib/hub.py similarity index 100% rename from hub.py rename to lib/hub.py diff --git a/keypacket.py b/lib/keypacket.py similarity index 100% rename from keypacket.py rename to lib/keypacket.py diff --git a/mysql.py b/lib/mysql.py similarity index 100% rename from mysql.py rename to lib/mysql.py diff --git a/passphrase.py b/lib/passphrase.py similarity index 100% rename from passphrase.py rename to lib/passphrase.py diff --git a/pathmap.py b/lib/pathmap.py similarity index 100% rename from pathmap.py rename to lib/pathmap.py diff --git a/pgsql.py b/lib/pgsql.py similarity index 100% rename from pgsql.py rename to lib/pgsql.py diff --git a/pkgman.py b/lib/pkgman.py similarity index 100% rename from pkgman.py rename to lib/pkgman.py diff --git a/registry.py b/lib/registry.py similarity index 99% rename from registry.py rename to lib/registry.py index 1ac7c2d..d6582c6 100644 --- a/registry.py +++ b/lib/registry.py @@ -15,7 +15,7 @@ import re from os.path import * from paths import Paths as _Paths -import simplejson +import json from datetime import datetime @@ -204,12 +204,12 @@ def backup_resume_conf(self, val=UNDEFINED): return try: - return BackupSessionConf(simplejson.loads(s)) + return BackupSessionConf(json.loads(s)) except: return else: - s = simplejson.dumps(val) + s = json.dumps(val) self._file_str(self.path.backup_resume, s) backup_resume_conf = property(backup_resume_conf, backup_resume_conf) diff --git a/restore.py b/lib/restore.py similarity index 98% rename from restore.py rename to lib/restore.py index ed7469b..d1556d6 100644 --- a/restore.py +++ b/lib/restore.py @@ -27,7 +27,7 @@ import mysql import pgsql -import simplejson +import json from temp import TempFile @@ -52,7 +52,7 @@ def __init__(self, backup_extract_path, limits=[], rollback=True, simulate=False if simulate: rollback = False - self.conf = AttrDict(simplejson.loads(file(self.extras.backup_conf).read())) \ + self.conf = AttrDict(json.loads(file(self.extras.backup_conf).read())) \ if exists(self.extras.backup_conf) else None self.simulate = simulate diff --git a/retry.py b/lib/retry.py similarity index 100% rename from retry.py rename to lib/retry.py diff --git a/rollback.py b/lib/rollback.py similarity index 100% rename from rollback.py rename to lib/rollback.py diff --git a/squid.py b/lib/squid.py similarity index 100% rename from squid.py rename to lib/squid.py diff --git a/userdb.py b/lib/userdb.py similarity index 100% rename from userdb.py rename to lib/userdb.py diff --git a/utils.py b/lib/utils.py similarity index 100% rename from utils.py rename to lib/utils.py diff --git a/version.py b/lib/version.py similarity index 100% rename from version.py rename to lib/version.py diff --git a/update-dep-versions.sh b/update-dep-versions.sh new file mode 100755 index 0000000..b596ea3 --- /dev/null +++ b/update-dep-versions.sh @@ -0,0 +1,21 @@ +#!/bin/bash -eu + + +tmp_file="$(mktemp -d)" + +COMMIT_ID_PATH="$(pwd)/dep-commit-ids" + +rm -f "$COMMIT_ID_PATH" + +cd "$tmp_file" +for pkg in python-crypto python-pycurl tklbam-duplicity turnkey-pylib tklbam-python-boto pycurl-wrapper; do + echo "# getting HEAD commit id for $pkg" + if [[ $pkg == "pycurl-wrapper" ]]; then + commit_id="$(git ls-remote "https://github.com/turnkeylinux/$pkg" | grep refs/heads/python2 | awk '{ print $1 }')" + else + commit_id="$(git ls-remote "https://github.com/turnkeylinux/$pkg" | grep HEAD | awk '{ print $1 }')" + fi + echo "$pkg:$commit_id" >> "$COMMIT_ID_PATH" +done + +rm -r "$tmp_file" diff --git a/update_pypy.sh b/update_pypy.sh new file mode 100755 index 0000000..7c90c0f --- /dev/null +++ b/update_pypy.sh @@ -0,0 +1,28 @@ +#!/bin/bash -eu + +# Run this script to update the version of pypy to be dynamically downloaded +# when building the tklbam deb package. Will always download the latest pypy. + +# pypy arch -> debian arch: +# - linux64 = amd64 +# - aarch64 = arm64 + +SUPPORTED_ARCH=(linux64 aarch64) + +tmp_file=$(mktemp) + +curl --silent https://pypy.org/checksums.html --output "$tmp_file" + +echo "# pypy versions and checksums - run ./update_pypy.sh to update" > \ + checksums.txt +for arch in "${SUPPORTED_ARCH[@]}"; do + sed -n "/pypy2\.7.*$arch/{s|<*.*>||;p;q;}" "$tmp_file" >> checksums.txt +done +rm -rf "$tmp_file" +if git diff --quiet HEAD -- checksums.txt; then + echo "no update available - checksums.txt unchanged" +else + echo "checksums.txt updated; double check and commit" + # non-zero exit if there's something to do + exit 1 +fi