From 67019cc2fd4098b2ae321465411ac6819e7c1d5f Mon Sep 17 00:00:00 2001 From: Robert Schade Date: Sun, 30 Jul 2017 17:26:57 +0200 Subject: [PATCH 01/16] Update README.md --- README.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/README.md b/README.md index 7e062cd..381db35 100644 --- a/README.md +++ b/README.md @@ -27,6 +27,13 @@ Tested and working on devices: Feedback on tests with other devices and android versions is very welcome. +Warning messages like +```` +WARNING: linker: /data/data/com.termux/files/usr/lib/libacl.so.1.1.0: unused DT entry: type 0xf arg 0x449 +```` +are due to the Android linker. More details can be found at https://stackoverflow.com/questions/33206409/unused-dt-entry-type-0x1d-arg. + + So all in all my Android backup setup looks like: - borg, termux and tasker - termux: Task (https://f-droid.org/packages/com.termux.tasker/) for tasker integration From 5a8e181886c04c1e555aa18cf19e448ed74e0234 Mon Sep 17 00:00:00 2001 From: Austen Adler Date: Sat, 29 Jul 2017 23:01:34 -0400 Subject: [PATCH 02/16] Add +x to scripts --- borg_ssh_wrapper | 0 build.sh | 0 2 files changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 borg_ssh_wrapper mode change 100644 => 100755 build.sh diff --git a/borg_ssh_wrapper b/borg_ssh_wrapper old mode 100644 new mode 100755 diff --git a/build.sh b/build.sh old mode 100644 new mode 100755 From 7e009c348547521e5588d77a45a8bc663bc4aa13 Mon Sep 17 00:00:00 2001 From: Austen Adler Date: Sat, 29 Jul 2017 23:02:22 -0400 Subject: [PATCH 03/16] Use $* over $1..$8 in borg_ssh_wrapper --- borg_ssh_wrapper | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/borg_ssh_wrapper b/borg_ssh_wrapper index f101187..b890be0 100755 --- a/borg_ssh_wrapper +++ b/borg_ssh_wrapper @@ -1,3 +1,3 @@ #!/data/data/com.termux/files/usr/bin/sh export LD_LIBRARY_PATH=/data/data/com.termux/files/usr/lib:/system/lib64/ -ssh $1 $2 $3 $4 $5 $6 $7 $8 +ssh $* From 97a2ca9f72d8698dad474a747126591df7f27045 Mon Sep 17 00:00:00 2001 From: wesley jackson Date: Sat, 16 Sep 2017 23:35:19 -0500 Subject: [PATCH 04/16] Update build.sh Fixed LDFLAGS --- build.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build.sh b/build.sh index d0a7d88..00aa796 100755 --- a/build.sh +++ b/build.sh @@ -36,7 +36,7 @@ sed -i "s/\/bin\/sh/\/data\/data\/com.termux\/files\/usr\/bin\/sh/" include/inst sed -i "s/TMPDIR=\/tmp/TMPDIR=tmp/g" config.guess mkdir tmp -./configure CC=clang LDFLAGS=-lintl --prefix=/data/data/com.termux/files/usr/ +./configure CC=clang --prefix=/data/data/com.termux/files/usr/ #fix for ./include/attr/xattr.h:37:58: error: expected function body after function declarator # const void *__value, size_t __size, int __flags) __THROW; @@ -60,7 +60,7 @@ sed -i "s/\/bin\/sh/\/data\/data\/com.termux\/files\/usr\/bin\/sh/" include/inst sed -i "s/TMPDIR=\/tmp/TMPDIR=tmp/g" config.guess mkdir tmp -./configure --prefix=/data/data/com.termux/files/usr/ CC=clang LDFLAGS=-lintl +./configure --prefix=/data/data/com.termux/files/usr/ CC=clang make make install install-lib install-dev cd .. From 2d154c2d3822bb4eb296611abe23ad50648d6946 Mon Sep 17 00:00:00 2001 From: Robert Schade Date: Sun, 17 Sep 2017 15:05:44 +0200 Subject: [PATCH 05/16] Update README.md added note about selinux and virtualenv --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 381db35..1f0ebaf 100644 --- a/README.md +++ b/README.md @@ -10,6 +10,7 @@ How to use: - open app - run "apt update; apt install git" - run "git clone https://github.com/ravenschade/borgbackup_on_android.git" + - set selinux to permissive (/system/bin/setenforce 1, otherwise virtualenv for python will not work properly) - run "cd borgbackup_on_android; bash build.sh" Tested with: From 228441d3b81087ec020a0ded18a29ac23bbeb855 Mon Sep 17 00:00:00 2001 From: Robert Schade Date: Tue, 24 Oct 2017 13:20:13 +0200 Subject: [PATCH 06/16] Update README.md modification of the readme based on correction by nathaningle. --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 1f0ebaf..b302e89 100644 --- a/README.md +++ b/README.md @@ -10,9 +10,9 @@ How to use: - open app - run "apt update; apt install git" - run "git clone https://github.com/ravenschade/borgbackup_on_android.git" - - set selinux to permissive (/system/bin/setenforce 1, otherwise virtualenv for python will not work properly) - run "cd borgbackup_on_android; bash build.sh" - +- (if virtualenv for python does not work properly you have to set selinux to permissive (do "/system/bin/setenforce 0" with root permissions)) + Tested with: - termux 0.53 - borg 1.0.12 From 80270131769eacf2b2c1c7df18b5d6a3cce6078a Mon Sep 17 00:00:00 2001 From: Robert Schade Date: Sat, 25 Nov 2017 19:53:26 +0100 Subject: [PATCH 07/16] changed to borg 1.1 --- build.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build.sh b/build.sh index 00aa796..3f31da3 100755 --- a/build.sh +++ b/build.sh @@ -9,8 +9,8 @@ source borg-env/bin/activate git clone https://github.com/borgbackup/borg.git cd borg -git branch 1.0-maint remotes/origin/1.0-maint -git checkout 1.0-maint +git branch 1.1-maint remotes/origin/1.1-maint +git checkout 1.1-maint pip install -r requirements.d/development.txt From 4b8ab7392b8ef0f9d9dcec81a150b77a69fa3a58 Mon Sep 17 00:00:00 2001 From: Robert Schade Date: Sat, 25 Nov 2017 20:12:52 +0100 Subject: [PATCH 08/16] openssl path --- README.md | 8 ++++---- borg.patch | 10 +++++++++- 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index b302e89..3d91581 100644 --- a/README.md +++ b/README.md @@ -14,8 +14,8 @@ How to use: - (if virtualenv for python does not work properly you have to set selinux to permissive (do "/system/bin/setenforce 0" with root permissions)) Tested with: -- termux 0.53 -- borg 1.0.12 +- termux 0.54 +- borg 1.1.0 Tested and working so far is: - creation of repositories @@ -23,8 +23,8 @@ Tested and working so far is: - backup up to remote repositories via ssh Tested and working on devices: -- Huawei Nexus6p with stock Android 7.1.2 (angler, aarch64) -- Samsung Galaxy Note 2 with Lineage 14.1 (n7100, armv7l) +- Huawei Nexus6p with stock Android 8.0.0 (angler, aarch64) +- Samsung Galaxy Note 2 with Lineage 14.1 (n7100, armv7l, only tested with borg 1.0) Feedback on tests with other devices and android versions is very welcome. diff --git a/borg.patch b/borg.patch index 3bb7fff..1173270 100644 --- a/borg.patch +++ b/borg.patch @@ -20,4 +20,12 @@ index 33116acd..40db200f 100644 if os.environ.get('BORG_LZ4_PREFIX'): possible_lz4_prefixes.insert(0, os.environ.get('BORG_LZ4_PREFIX')) lz4_prefix = detect_lz4(possible_lz4_prefixes) - +@@ -161,7 +161,7 @@ def detect_libb2(prefixes): + crypto_libraries = ['crypto'] + + possible_openssl_prefixes = ['/usr', '/usr/local', '/usr/local/opt/openssl', '/usr/local/ssl', '/usr/local/openssl', +- '/usr/local/borg', '/opt/local', '/opt/pkg', ] ++ '/usr/local/borg', '/opt/local', '/opt/pkg', '/data/data/com.termux/files/usr/'] + if os.environ.get('BORG_OPENSSL_PREFIX'): + possible_openssl_prefixes.insert(0, os.environ.get('BORG_OPENSSL_PREFIX')) + ssl_prefix = detect_openssl(possible_openssl_prefixes) From a62732b506007d21d85d7e8bf5329d2962b28a43 Mon Sep 17 00:00:00 2001 From: Robert Schade Date: Sat, 25 Nov 2017 20:29:18 +0100 Subject: [PATCH 09/16] environment variables instead of patch --- borg.patch | 31 ------------------------------- build.sh | 5 +++-- 2 files changed, 3 insertions(+), 33 deletions(-) delete mode 100644 borg.patch diff --git a/borg.patch b/borg.patch deleted file mode 100644 index 1173270..0000000 --- a/borg.patch +++ /dev/null @@ -1,31 +0,0 @@ -diff --git setup.py setup.py -index 33116acd..40db200f 100644 ---- setup.py -+++ setup.py -@@ -115,7 +115,7 @@ def detect_lz4(prefixes): - include_dirs = [] - library_dirs = [] - --possible_openssl_prefixes = ['/usr', '/usr/local', '/usr/local/opt/openssl', '/usr/local/ssl', '/usr/local/openssl', '/usr/local/borg', '/opt/local'] -+possible_openssl_prefixes = ['/usr', '/usr/local', '/usr/local/opt/openssl', '/usr/local/ssl', '/usr/local/openssl', '/usr/local/borg', '/opt/local', '/data/data/com.termux/files/usr/'] - if os.environ.get('BORG_OPENSSL_PREFIX'): - possible_openssl_prefixes.insert(0, os.environ.get('BORG_OPENSSL_PREFIX')) - ssl_prefix = detect_openssl(possible_openssl_prefixes) -@@ -125,7 +125,7 @@ def detect_lz4(prefixes): - library_dirs.append(os.path.join(ssl_prefix, 'lib')) - - --possible_lz4_prefixes = ['/usr', '/usr/local', '/usr/local/opt/lz4', '/usr/local/lz4', '/usr/local/borg', '/opt/local'] -+possible_lz4_prefixes = ['/usr', '/usr/local', '/usr/local/opt/lz4', '/usr/local/lz4', '/usr/local/borg', '/opt/local', '/data/data/com.termux/files/usr/'] - if os.environ.get('BORG_LZ4_PREFIX'): - possible_lz4_prefixes.insert(0, os.environ.get('BORG_LZ4_PREFIX')) - lz4_prefix = detect_lz4(possible_lz4_prefixes) -@@ -161,7 +161,7 @@ def detect_libb2(prefixes): - crypto_libraries = ['crypto'] - - possible_openssl_prefixes = ['/usr', '/usr/local', '/usr/local/opt/openssl', '/usr/local/ssl', '/usr/local/openssl', -- '/usr/local/borg', '/opt/local', '/opt/pkg', ] -+ '/usr/local/borg', '/opt/local', '/opt/pkg', '/data/data/com.termux/files/usr/'] - if os.environ.get('BORG_OPENSSL_PREFIX'): - possible_openssl_prefixes.insert(0, os.environ.get('BORG_OPENSSL_PREFIX')) - ssl_prefix = detect_openssl(possible_openssl_prefixes) diff --git a/build.sh b/build.sh index 3f31da3..0c20450 100755 --- a/build.sh +++ b/build.sh @@ -65,8 +65,9 @@ make make install install-lib install-dev cd .. -#patching paths in setup.py -patch -p0 < ../borg.patch +#patching paths +export BORG_OPENSSL_PREFIX="/data/data/com.termux/files/usr/" +export BORG_LZ4_PREFIX="/data/data/com.termux/files/usr/" pip install -e . From c182a192902f1529de0c48229b5430fe4389d123 Mon Sep 17 00:00:00 2001 From: Robert Schade Date: Sat, 25 Nov 2017 20:37:51 +0100 Subject: [PATCH 10/16] updated version numbers --- README.md | 6 +++--- build.sh | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 3d91581..2ad0e3e 100644 --- a/README.md +++ b/README.md @@ -14,8 +14,8 @@ How to use: - (if virtualenv for python does not work properly you have to set selinux to permissive (do "/system/bin/setenforce 0" with root permissions)) Tested with: -- termux 0.54 -- borg 1.1.0 +- termux 0.56 +- borg 1.1.3 Tested and working so far is: - creation of repositories @@ -24,7 +24,7 @@ Tested and working so far is: Tested and working on devices: - Huawei Nexus6p with stock Android 8.0.0 (angler, aarch64) -- Samsung Galaxy Note 2 with Lineage 14.1 (n7100, armv7l, only tested with borg 1.0) +- Samsung Galaxy Note 2 with Lineage 14.1 (n7100, armv7l, Android 7.1.1) Feedback on tests with other devices and android versions is very welcome. diff --git a/build.sh b/build.sh index 0c20450..5ebfe15 100755 --- a/build.sh +++ b/build.sh @@ -79,7 +79,7 @@ cp borg_ssh_wrapper /data/data/com.termux/files/usr/bin/borg_ssh_wrapper chmod +x /data/data/com.termux/files/usr/bin/borg_ssh_wrapper #test by creating a backup of the borg directory -borg init borg_test +borg init -e none borg_test borg create borg_test::1 borg borg list borg_test borg info borg_test::1 From 565c61a0adba1d996d0db5f68d2db0023c6741ee Mon Sep 17 00:00:00 2001 From: Robert Schade Date: Sun, 26 Nov 2017 00:01:17 +0100 Subject: [PATCH 11/16] check for sync_file_range and patch --- README.md | 9 ++++++--- borg_sync_file_range.patch | 16 ++++++++++++++++ build.sh | 10 ++++++++++ sync_file_range_test/setup.py | 6 ++++++ sync_file_range_test/sync_file_range_test.py | 6 ++++++ sync_file_range_test/sync_file_range_test.pyx | 11 +++++++++++ sync_file_range_test/test.sh | 5 +++++ 7 files changed, 60 insertions(+), 3 deletions(-) create mode 100644 borg_sync_file_range.patch create mode 100644 sync_file_range_test/setup.py create mode 100644 sync_file_range_test/sync_file_range_test.py create mode 100644 sync_file_range_test/sync_file_range_test.pyx create mode 100644 sync_file_range_test/test.sh diff --git a/README.md b/README.md index 2ad0e3e..5f149ef 100644 --- a/README.md +++ b/README.md @@ -11,11 +11,14 @@ How to use: - run "apt update; apt install git" - run "git clone https://github.com/ravenschade/borgbackup_on_android.git" - run "cd borgbackup_on_android; bash build.sh" -- (if virtualenv for python does not work properly you have to set selinux to permissive (do "/system/bin/setenforce 0" with root permissions)) - + - (if virtualenv for python does not work properly you have to set selinux to permissive (do "/system/bin/setenforce 0" with root permissions)) + +Known issues: + - borg starting at 1.1 requires the system call sync_file_range (see https://github.com/borgbackup/borg/pull/985 and https://github.com/borgbackup/borg/issues/1961). The linux subsystem in Windows 10 and some older Android versions (for example Lineage including 14.1) do not yet have this. Lineage 15.0 has this call and should work. I have added a test to the build script that checks if sync_file_range is available. If it is not, then I apply a patch (borg_sync_file_range.patch) that replaces this sync with convential file syncs. + Tested with: - termux 0.56 -- borg 1.1.3 +- borg 1.0.12, 1.1.3 Tested and working so far is: - creation of repositories diff --git a/borg_sync_file_range.patch b/borg_sync_file_range.patch new file mode 100644 index 0000000..9813709 --- /dev/null +++ b/borg_sync_file_range.patch @@ -0,0 +1,16 @@ +diff --git a/src/borg/platform/linux.pyx b/src/borg/platform/linux.pyx +index 25f71fa1..42ffa85f 100644 +--- a/src/borg/platform/linux.pyx ++++ b/src/borg/platform/linux.pyx +@@ -225,8 +225,9 @@ def acl_set(path, item, numeric_owner=False): + cdef _sync_file_range(fd, offset, length, flags): + assert offset & PAGE_MASK == 0, "offset %d not page-aligned" % offset + assert length & PAGE_MASK == 0, "length %d not page-aligned" % length +- if sync_file_range(fd, offset, length, flags) != 0: +- raise OSError(errno.errno, os.strerror(errno.errno)) ++ os.fdatasync(fd) ++ #if sync_file_range(fd, offset, length, flags) != 0: ++ # raise OSError(errno.errno, os.strerror(errno.errno)) + safe_fadvise(fd, offset, length, 'DONTNEED') + + cdef unsigned PAGE_MASK = sysconf(_SC_PAGESIZE) - 1 diff --git a/build.sh b/build.sh index 5ebfe15..b7a1f27 100755 --- a/build.sh +++ b/build.sh @@ -14,6 +14,16 @@ git checkout 1.1-maint pip install -r requirements.d/development.txt +#find if sync_file_range is available +s=`bash ../sync_file_range_test/test.sh` +if [ "$s" = "1" ]; +then + echo "patching borg to not use sync_file_range...." + git apply ../borg_sync_file_range.patch +else + echo "no need to patch borg" +fi + #download and build lz4 wget https://github.com/lz4/lz4/archive/v1.7.5.tar.gz -O lz4.tar.gz tar -xf lz4.tar.gz diff --git a/sync_file_range_test/setup.py b/sync_file_range_test/setup.py new file mode 100644 index 0000000..2650992 --- /dev/null +++ b/sync_file_range_test/setup.py @@ -0,0 +1,6 @@ +# setup.py - unnecessary if not redistributing the code, see below +from setuptools import setup +from Cython.Build import cythonize + +setup(name = 'Hello world app', + ext_modules = cythonize("*.pyx")) diff --git a/sync_file_range_test/sync_file_range_test.py b/sync_file_range_test/sync_file_range_test.py new file mode 100644 index 0000000..30af0f2 --- /dev/null +++ b/sync_file_range_test/sync_file_range_test.py @@ -0,0 +1,6 @@ +try: + import sync_file_range_test +except ImportError: + raise ImportError('sync_file_range not existent') + +sync_file_range_test.sync_file_range_test(0) diff --git a/sync_file_range_test/sync_file_range_test.pyx b/sync_file_range_test/sync_file_range_test.pyx new file mode 100644 index 0000000..ff4bb9e --- /dev/null +++ b/sync_file_range_test/sync_file_range_test.pyx @@ -0,0 +1,11 @@ + +cdef extern from "fcntl.h": + int sync_file_range(int fd, int offset, int nbytes, unsigned int flags) + +def sync_file_range_test(int x): + print("sync_file_range exists\n") + if x==-1: + sync_file_range(1,1,1,1) + + + diff --git a/sync_file_range_test/test.sh b/sync_file_range_test/test.sh new file mode 100644 index 0000000..d61e36d --- /dev/null +++ b/sync_file_range_test/test.sh @@ -0,0 +1,5 @@ +cd ../sync_file_range_test/ +python setup.py build_ext --inplace > setup.log 2> setup.err +python sync_file_range_test.py > test.log 2> test.err +grep "sync_file_range not existent" test.err | tail -n 1 | wc -l + From 0d64999cf8d36827ed93414076962483131c9e2f Mon Sep 17 00:00:00 2001 From: Robert Schade Date: Thu, 7 Dec 2017 17:50:26 +0100 Subject: [PATCH 12/16] Create borg.sh --- borg.sh | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 borg.sh diff --git a/borg.sh b/borg.sh new file mode 100644 index 0000000..7474b83 --- /dev/null +++ b/borg.sh @@ -0,0 +1,9 @@ +#!/data/data/com.termux/files/usr/bin/bash +t=`date +%d_%m_%Y` +export BORG_UNKNOWN_UNENCRYPTED_REPO_ACCESS_IS_OK=yes +export BORG_RELOCATED_REPO_ACCESS_IS_OK=yes +host=angler +dirs="/ /system /vendor /cache /persist /firmware /storage /data" +export BORG_RSH=borg_ssh_wrapper +source /data/data/com.termux/files/home/borgbackup_on_android/borg-env/bin/activate +borg create -C lz4 -p -v --stats --one-file-system backup:/backup/borg/$host::$t $dirs # 2> ~/borg_backup_${t}.err From abdac11ac9d0eb3ac068465e2f79da632f14a5e2 Mon Sep 17 00:00:00 2001 From: Robert Schade Date: Thu, 7 Dec 2017 17:52:28 +0100 Subject: [PATCH 13/16] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 5f149ef..48cf1ba 100644 --- a/README.md +++ b/README.md @@ -26,7 +26,7 @@ Tested and working so far is: - backup up to remote repositories via ssh Tested and working on devices: -- Huawei Nexus6p with stock Android 8.0.0 (angler, aarch64) +- Huawei Nexus6p with stock Android 8.1.0 (angler, aarch64) - Samsung Galaxy Note 2 with Lineage 14.1 (n7100, armv7l, Android 7.1.1) Feedback on tests with other devices and android versions is very welcome. From 9594a9f9d98c21a5e847f29c13bf808af5744e90 Mon Sep 17 00:00:00 2001 From: Austen Adler Date: Thu, 5 Jul 2018 14:45:50 -0400 Subject: [PATCH 14/16] Add notes to build.sh --- build.sh | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/build.sh b/build.sh index b7a1f27..1645ed0 100755 --- a/build.sh +++ b/build.sh @@ -1,17 +1,20 @@ #!/data/data/com.termux/files/usr/bin/bash set -x -apt -y install make clang openssl-dev perl tsu wget git python python-dev gnupg2 dirmngr curl autoconf automake sed gettext gzip pkg-config +set -e +apt -y install make clang openssl-dev perl tsu wget git python python-dev gnupg2 dirmngr curl autoconf automake sed gettext gzip pkg-config libcrypt-dev +# To make other files +apt -y install gettext-dev pip install virtualenv virtualenv --python=python3 borg-env source borg-env/bin/activate -git clone https://github.com/borgbackup/borg.git +test -d borg || git clone https://github.com/borgbackup/borg.git cd borg -git branch 1.1-maint remotes/origin/1.1-maint -git checkout 1.1-maint +git checkout 1.1.6 +pip install Cython pip install -r requirements.d/development.txt #find if sync_file_range is available From a16190dbdd8a33c5927ca7e633733604fdd1a2aa Mon Sep 17 00:00:00 2001 From: Austen Adler Date: Thu, 5 Jul 2018 14:46:38 -0400 Subject: [PATCH 15/16] Add gitignore --- .gitignore | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 .gitignore diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..7e0b07b --- /dev/null +++ b/.gitignore @@ -0,0 +1,4 @@ +sync_file_range_test +borg +borg_test +borg-env From ecae30b8b6f30fb33687192f6d655ab1d22180ba Mon Sep 17 00:00:00 2001 From: Austen Adler Date: Thu, 5 Jul 2018 14:47:02 -0400 Subject: [PATCH 16/16] Remove extra file --- build.sh | 2 -- 1 file changed, 2 deletions(-) diff --git a/build.sh b/build.sh index 1645ed0..115252e 100755 --- a/build.sh +++ b/build.sh @@ -2,8 +2,6 @@ set -x set -e apt -y install make clang openssl-dev perl tsu wget git python python-dev gnupg2 dirmngr curl autoconf automake sed gettext gzip pkg-config libcrypt-dev -# To make other files -apt -y install gettext-dev pip install virtualenv