fix bug in aging view, add man pages

diff --git a/Makefile b/Makefile
index e26de514cc4351f9445e336ea71c77533f764d4b..5334ad8504f842ac11a05cc09545823fc32e49c4 100644 (file)
--- a/Makefile
+++ b/Makefile
@@ -35,7 +35,7 @@ SCRIPTS=zpm zpm-install zpm-merge zpm-list zpm-preserve zpm-test zpm-log \
        zpm-contents zpm-uninstall zpm-pathmod zpm-rmpackage zpm-newpackage \
        zpm-pkg zpm-pkgfile zpm-gc zpm-repo zpm-update zpm-confgit
 
        zpm-contents zpm-uninstall zpm-pathmod zpm-rmpackage zpm-newpackage \
        zpm-pkg zpm-pkgfile zpm-gc zpm-repo zpm-update zpm-confgit
 

-MANPAGES=doc/zpm.8 $(addprefix doc/zpm-, list.8 contents.8 hash.8 quote.8 pathmod.8 note.8 vercmp.8 repo.8)
+MANPAGES=doc/zpm.8 $(addprefix doc/zpm-, list.8 contents.8 hash.8 quote.8 pathmod.8 note.8 vercmp.8 repo.8 gc.8)

 COMPILED=$(ZPKGBIN)
 PROGRAMS=$(SCRIPTS) $(COMPILED)
 
 COMPILED=$(ZPKGBIN)
 PROGRAMS=$(SCRIPTS) $(COMPILED)
 


@@ -164,7 +164,7 @@ zpm-add: zpm-add.o libzpm.a libelf.a
 zpm-quote: zpm-quote.o
        $(CC) $(CFLAGS) $(LDFLAGS) -o $@ $<
 
 zpm-quote: zpm-quote.o
        $(CC) $(CFLAGS) $(LDFLAGS) -o $@ $<
 

-db.sql: schema/main.sql schema/syncconflicts.sql schema/syncinfo.sql
+db.sql: schema/main.sql schema/syncconflicts.sql schema/syncinfo.sql schema/preserve_packages.sql schema/age.sql

        echo 'begin;' > $@
        cat $+ >> $@
        echo 'commit;' >> $@
        echo 'begin;' > $@
        cat $+ >> $@
        echo 'commit;' >> $@

diff --git a/doc/zpm-gc.8 b/doc/zpm-gc.8
new file mode 100644 (file)
index 0000000..f35fd79
--- /dev/null
+++ b/doc/zpm-gc.8
@@ -0,0 +1,78 @@
+.TH zpm-gc 8 2018-12-10 "ZPM 0.4"
+.SH NAME
+zpm-gc \- garbage collect a database
+.SH SYNOPSIS
+.B zpm gc [-f \fIdb\fB] [-cCP] [-l retention] [-n retention]
+.SH DESCRIPTION
+\fBzpm-gc\fR removes unneeded or unwanted elements from a zpm database.
+Actions taken will be logged in the database and written to stdout.
+Each action is described below.
+.SS Old packages
+Unused packages will be removed, unless they need to be retained to keep a
+library installed, or if they are within the keep target.  By default, two old
+versions of a package will be kept.  This phase is run by default,
+but can be suppressed with the \-P option.  The keep target can
+be set with the -k option or the ZPM_KEEP_PACKAGES environment
+variable.  If both are used, the command line option takes precedence.
+.PP
+Packages kept by the keep target do not include packages otherwise retained for
+library dependencies, so it is possible that more than the keep target older
+packages will be retained.  Only packages in an "updated" or "removed" status
+are eligible for garbage collection, so an admin can keep any arbitrary package
+from being garbage collected by setting the status of the package to some other
+value.
+.SS Log entries
+Old log entries can be removed by passing the \-l option.  The -l
+option takes an argument specifying which log entries to delete.
+.SS Notes
+Notes can be removed with the \-n option.
+.SS Unreferenced file content
+File content unreferenced by either a package, note, or script
+will be removed by default.  This action can be suppressed with \-C.
+.SS Package file content
+File content belonging to packages can be removed with the \-F option.
+This is not done by default, and the action can be restricted to
+a specific package with the \-p option.
+.SS Compact
+Re-write the database file out completely to minimize the amount
+of disk space it uses.  This option can take a few minutes
+for large (gigabyte sized) databases, and is not run by default.
+It can be enabled with the \-c option.
+.SH OPTIONS
+.TP
+\-f \fIdbfile\fR
+Use \fIdbfile\fR as the database to run against, instead of the default.
+.TP
+\-l \fIretention\fR
+Delete old logentries that correspond to the retention.
+A retention starting with a '+' is taken as the number of days to retain.
+A retention containing only digits is taken as a number of log entries to
+retain, anything else is interpreted as a timestamp.
+.TP
+\-n \fIretention\fR
+Delete old notes that correspond to the retention.
+A retention is interpreted the same way as for the \-l option.
+.TP
+\-z
+Zero out the set of actions to preform.
+Specific actions can be specified as options after the \-z.
+.TP
+\-k \fIn\fR
+Set the number of old packages to retain to \fIn\fR.  These are in addition to
+any packages retained because they contain libraries that are still needed.
+.SH EXAMPLES
+.PP
+.nf
+zpm-gc
+.fi
+.SH EXIT STATUS
+0 if successful, non-zero if there was an error
+.SH FILES
+None
+.SH ENVIRONMENT
+ZPMDB ZPM_KEEP_PACKAGES
+.SH AUTHOR
+Nathan Wagner
+.SH SEE ALSO
+.BR zpm (8)
+.BR zpm-log (8)

diff --git a/doc/zpm-update.8 b/doc/zpm-update.8
index 638b06a740eaa3907ca3e5ce946b7feb79409b08..96e700bb7e9315e5ad409e0e83f54bb5ace8ffe4 100644 (file)
--- a/doc/zpm-update.8
+++ b/doc/zpm-update.8
@@ -36,6 +36,11 @@ In other words, all of the packages given are updated, removed, installed at
 the same time, rather than taking each package one at a time and installing,
 updating, or removing it.  If the admin wants to do the packages one at a time,
 zpm-update can be run multiple times.
 the same time, rather than taking each package one at a time and installing,
 updating, or removing it.  If the admin wants to do the packages one at a time,
 zpm-update can be run multiple times.

+.SS Package Search
+Packages will be searched for in *.repo files, the local database, and
+in individual zpm package files.  These are looked for by default
+in /var/lib/zpm/*.repo /var/lib/zpm/packages/*.zpm and the usual
+location for the local database.

 .SH OPTIONS
 .TP
 \-f
 .SH OPTIONS
 .TP
 \-f


@@ -48,7 +53,7 @@ and no files are changed, this can only detect and report some
 types of errors.
 .TP
 \-L
 types of errors.
 .TP
 \-L

-Don't check for library dependencies
+Don't check for library dependencies, passed on to zpm-search.

 .TP
 \-A
 run zpm-syncfs in accept mode
 .TP
 \-A
 run zpm-syncfs in accept mode


@@ -83,6 +88,7 @@ ZPMDB
 Nathan Wagner
 .SH SEE ALSO
 .BR zpm (8)
 Nathan Wagner
 .SH SEE ALSO
 .BR zpm (8)

+.BR zpm-search (8)

 .BR zpm-script (8)
 .BR zpm-repo (8)
 .BR zpm-syncfs (8)
 .BR zpm-script (8)
 .BR zpm-repo (8)
 .BR zpm-syncfs (8)

diff --git a/schema/age.sql b/schema/age.sql
new file mode 100644 (file)
index 0000000..bdc5da8
--- /dev/null
+++ b/schema/age.sql
@@ -0,0 +1,24 @@
+create view package_age as
+with aging as (
+select
+P.package, P.version,P.release,P.status,
+row_number() over (
+       partition by P.package
+       order by P.version collate vercmp desc, P.release desc
+) as age
+from
+packages P
+left join preserve_packages KEEP
+on KEEP.package = P.package and KEEP.version = P.version and KEEP.release = P.release
+where P.status in ('updated','removed')
+and KEEP.package is null
+)
+select * from aging
+union
+select P.package, P.version, P.release, P.status, 0 as age
+from packages P where P.status = 'installed'
+union
+select P.package, P.version, P.release, 'preserved' as status, 0 as age
+from preserve_packages P
+order by package, version collate vercmp desc, release desc
+;

diff --git a/zpm-gc b/zpm-gc
index 08ba1e19c85f73446da5472d9c4d20defca279c2..d2d49fc861c0dd95472395f13be317312f2b1410 100755 (executable)
--- a/zpm-gc
+++ b/zpm-gc
@@ -1,5 +1,6 @@
 #!/bin/sh
 
 #!/bin/sh
 

+set -e

 # garbage collect a package database
 # flags to skip phases
 
 # garbage collect a package database
 # flags to skip phases
 


@@ -20,6 +21,7 @@ remove_orphaned_elf=1
 # -fF remove failed
 remove_failed=1
 # -hH fix file content hash
 # -fF remove failed
 remove_failed=1
 # -hH fix file content hash

+remove_incomplete_packages=0

 fix_hash=0
 # -hH remove install history by date
 remove_history=
 fix_hash=0
 # -hH remove install history by date
 remove_history=


@@ -27,29 +29,44 @@ remove_history=
 remove_logs=
 # -nN remove acked notes
 remove_acked_notes=0
 remove_logs=
 # -nN remove acked notes
 remove_acked_notes=0

-# -oO remove outdated packages, unless preserved
-remove_old_packages=0

 # -pP remove history for missing packages
 clean_missing_package_history=0
 # -pP remove history for missing packages
 clean_missing_package_history=0

-# -rR remove orphaned rows in files table packagefiles, notes, scripts, ...
-remove_orphaned_files=1
+

 # -hH remove all file content that is not scripts or configs
 remove_packagefile_content=1
 remove_script_content=1
 remove_config_content=1
 # -rR remove removed packages, unless preserved
 remove_removed_packages=0
 # -hH remove all file content that is not scripts or configs
 remove_packagefile_content=1
 remove_script_content=1
 remove_config_content=1
 # -rR remove removed packages, unless preserved
 remove_removed_packages=0

+

 # -vV compactify the database file
 # -vV compactify the database file

-vacuum=1
+vacuum=0
+
+# remove orphaned rows in files table packagefiles, notes, scripts, ...
+unreferenced_file_content=1
+
+# remove outdated packages, unless preserved
+remove_old_packages=1

 
 db=${ZPMDB:-/var/lib/zpm/local.db}
 
 
 db=${ZPMDB:-/var/lib/zpm/local.db}
 

-while getopts 'f:'; do
+retain_old=${ZPM_KEEP_PACKAGES:-2}
+
+while getopts 'f:PcCk:' opt; do

        case $opt in
                f) db="$OPTARG" ;;
        case $opt in
                f) db="$OPTARG" ;;

+               P) remove_old_packages=0 ;;
+               c) vacuum=1 ;;
+               C) unreferenced_file_content=0 ;;
+               k) retain_old="$OPTARG" ;;

        esac
 done
 
        esac
 done
 

+retain_old=$((retain_old + 0))
+if [ $retain_old -le 0 ]; then
+       retain_old=0
+fi
+

 if [ -z "$db" ]; then
        echo must specify database file
        exit 1
 if [ -z "$db" ]; then
        echo must specify database file
        exit 1


@@ -65,10 +82,12 @@ export ZPMDB
 # remove failed packages
 for pkg in $(zpm list -s failed); do
        zpm rmpackage -m 'gc removed failed' "$pkg"
 # remove failed packages
 for pkg in $(zpm list -s failed); do
        zpm rmpackage -m 'gc removed failed' "$pkg"

+       echo removed $pkg

 done
 
 for pkg in $(zpm list -s dryrun); do
        zpm rmpackage -m 'gc removed dryrun' "$pkg"
 done
 
 for pkg in $(zpm list -s dryrun); do
        zpm rmpackage -m 'gc removed dryrun' "$pkg"

+       echo removed $pkg

 done
 
 # remove incomplete packages
 done
 
 # remove incomplete packages


@@ -76,24 +95,27 @@ done
 if [ $remove_incomplete_packages -ne 0 ]; then
        for pkg in $(zpm list -F 'hash is null'); do
                zpm rmpackage -m 'gc removed incomplete' $pkg
 if [ $remove_incomplete_packages -ne 0 ]; then
        for pkg in $(zpm list -F 'hash is null'); do
                zpm rmpackage -m 'gc removed incomplete' $pkg

+       echo removed $pkg

        done
 fi
 
 # remove old version packages, unless preserved
        done
 fi
 
 # remove old version packages, unless preserved

-# TODO need 'rstatus' flag
+# remove removed packages, unless preserved

 if [ $remove_old_packages -ne 0 ]; then
 if [ $remove_old_packages -ne 0 ]; then

-       # TODO check for preserved
-       # TODO add ability to keep a certain number of back packages
-       zpm shell $ZPMDB "delete from packages where status = 'updated'"
+       zpm shell $ZPMDB "select printf('%s-%s-%s',package,version,release) from package_age where age > $retain_old" | while read pkg; do
+       zpm rmpackage -m 'gc removed old package' $pkg
+       echo removed $pkg
+done

 fi
 
 fi
 

-# remove removed packages, unless preserved
-

 # remove orphaned rows in files table
 # references in packagefiles, notes, scripts, possibly others
 
 if [ $remove_orphaned_files -ne 0 ]; then
 # remove orphaned rows in files table
 # references in packagefiles, notes, scripts, possibly others
 
 if [ $remove_orphaned_files -ne 0 ]; then

-       zpm shell $ZPMDB 'delete from files where hash in (select hash from filerefs where refcount = 0);'
+       count=$(zpm shell $ZPMDB 'delete from files where hash in (select hash from filerefs where refcount = 0);select changes()')
+       if [ $count -gt 0 ]; then
+               zpm log -v -i -a 'gc' -t $ZPMDB "removed $count unreferenced content"
+       fi

 fi
 
 # remove old logs, given by date
 fi
 
 # remove old logs, given by date


@@ -107,17 +129,18 @@ fi
 
 # remove history for missing packages
 if [ $clean_missing_package_history -ne 0 ]; then
 
 # remove history for missing packages
 if [ $clean_missing_package_history -ne 0 ]; then

-
+       # delete from zpmlog where ?
+       true

 fi
 
 # remove command history by date
 
 fi
 
 # remove command history by date
 

+# compactify the database file

 if [ $vacuum -ne 0 ]; then
 if [ $vacuum -ne 0 ]; then

-       # compactify the database file

        osize=$(zpm stat -f '%s' $ZPMDB)
        osize=$(zpm stat -f '%s' $ZPMDB)

-       zpm log -v -i -a 'gc compacting' -t $ZPMDB

        zpm shell $ZPMDB vacuum
        nsize=$(zpm stat -f '%s' $ZPMDB)
        zpm shell $ZPMDB vacuum
        nsize=$(zpm stat -f '%s' $ZPMDB)

+       zpm log -v -i -a 'gc compacted' -t $ZPMDB "$osize -> $nsize"

 
        printf 'original size %s\n' $osize
        printf 'new size %s\n' $nsize
 
        printf 'original size %s\n' $osize
        printf 'new size %s\n' $nsize