From e391f268e77b1609f4df34b1bb272cad3b5de7d4 Mon Sep 17 00:00:00 2001 From: Nathan Wagner Date: Fri, 21 Dec 2018 01:30:28 +0000 Subject: [PATCH] fix bug in aging view, add man pages --- Makefile | 4 +-- doc/zpm-gc.8 | 78 ++++++++++++++++++++++++++++++++++++++++++++++++ doc/zpm-update.8 | 8 ++++- schema/age.sql | 24 +++++++++++++++ zpm-gc | 55 ++++++++++++++++++++++++---------- 5 files changed, 150 insertions(+), 19 deletions(-) create mode 100644 doc/zpm-gc.8 create mode 100644 schema/age.sql diff --git a/Makefile b/Makefile index e26de51..5334ad8 100644 --- a/Makefile +++ b/Makefile @@ -35,7 +35,7 @@ SCRIPTS=zpm zpm-install zpm-merge zpm-list zpm-preserve zpm-test zpm-log \ zpm-contents zpm-uninstall zpm-pathmod zpm-rmpackage zpm-newpackage \ zpm-pkg zpm-pkgfile zpm-gc zpm-repo zpm-update zpm-confgit -MANPAGES=doc/zpm.8 $(addprefix doc/zpm-, list.8 contents.8 hash.8 quote.8 pathmod.8 note.8 vercmp.8 repo.8) +MANPAGES=doc/zpm.8 $(addprefix doc/zpm-, list.8 contents.8 hash.8 quote.8 pathmod.8 note.8 vercmp.8 repo.8 gc.8) COMPILED=$(ZPKGBIN) PROGRAMS=$(SCRIPTS) $(COMPILED) @@ -164,7 +164,7 @@ zpm-add: zpm-add.o libzpm.a libelf.a zpm-quote: zpm-quote.o $(CC) $(CFLAGS) $(LDFLAGS) -o $@ $< -db.sql: schema/main.sql schema/syncconflicts.sql schema/syncinfo.sql +db.sql: schema/main.sql schema/syncconflicts.sql schema/syncinfo.sql schema/preserve_packages.sql schema/age.sql echo 'begin;' > $@ cat $+ >> $@ echo 'commit;' >> $@ diff --git a/doc/zpm-gc.8 b/doc/zpm-gc.8 new file mode 100644 index 0000000..f35fd79 --- /dev/null +++ b/doc/zpm-gc.8 @@ -0,0 +1,78 @@ +.TH zpm-gc 8 2018-12-10 "ZPM 0.4" +.SH NAME +zpm-gc \- garbage collect a database +.SH SYNOPSIS +.B zpm gc [-f \fIdb\fB] [-cCP] [-l retention] [-n retention] +.SH DESCRIPTION +\fBzpm-gc\fR removes unneeded or unwanted elements from a zpm database. +Actions taken will be logged in the database and written to stdout. +Each action is described below. +.SS Old packages +Unused packages will be removed, unless they need to be retained to keep a +library installed, or if they are within the keep target. By default, two old +versions of a package will be kept. This phase is run by default, +but can be suppressed with the \-P option. The keep target can +be set with the -k option or the ZPM_KEEP_PACKAGES environment +variable. If both are used, the command line option takes precedence. +.PP +Packages kept by the keep target do not include packages otherwise retained for +library dependencies, so it is possible that more than the keep target older +packages will be retained. Only packages in an "updated" or "removed" status +are eligible for garbage collection, so an admin can keep any arbitrary package +from being garbage collected by setting the status of the package to some other +value. +.SS Log entries +Old log entries can be removed by passing the \-l option. The -l +option takes an argument specifying which log entries to delete. +.SS Notes +Notes can be removed with the \-n option. +.SS Unreferenced file content +File content unreferenced by either a package, note, or script +will be removed by default. This action can be suppressed with \-C. +.SS Package file content +File content belonging to packages can be removed with the \-F option. +This is not done by default, and the action can be restricted to +a specific package with the \-p option. +.SS Compact +Re-write the database file out completely to minimize the amount +of disk space it uses. This option can take a few minutes +for large (gigabyte sized) databases, and is not run by default. +It can be enabled with the \-c option. +.SH OPTIONS +.TP +\-f \fIdbfile\fR +Use \fIdbfile\fR as the database to run against, instead of the default. +.TP +\-l \fIretention\fR +Delete old logentries that correspond to the retention. +A retention starting with a '+' is taken as the number of days to retain. +A retention containing only digits is taken as a number of log entries to +retain, anything else is interpreted as a timestamp. +.TP +\-n \fIretention\fR +Delete old notes that correspond to the retention. +A retention is interpreted the same way as for the \-l option. +.TP +\-z +Zero out the set of actions to preform. +Specific actions can be specified as options after the \-z. +.TP +\-k \fIn\fR +Set the number of old packages to retain to \fIn\fR. These are in addition to +any packages retained because they contain libraries that are still needed. +.SH EXAMPLES +.PP +.nf +zpm-gc +.fi +.SH EXIT STATUS +0 if successful, non-zero if there was an error +.SH FILES +None +.SH ENVIRONMENT +ZPMDB ZPM_KEEP_PACKAGES +.SH AUTHOR +Nathan Wagner +.SH SEE ALSO +.BR zpm (8) +.BR zpm-log (8) diff --git a/doc/zpm-update.8 b/doc/zpm-update.8 index 638b06a..96e700b 100644 --- a/doc/zpm-update.8 +++ b/doc/zpm-update.8 @@ -36,6 +36,11 @@ In other words, all of the packages given are updated, removed, installed at the same time, rather than taking each package one at a time and installing, updating, or removing it. If the admin wants to do the packages one at a time, zpm-update can be run multiple times. +.SS Package Search +Packages will be searched for in *.repo files, the local database, and +in individual zpm package files. These are looked for by default +in /var/lib/zpm/*.repo /var/lib/zpm/packages/*.zpm and the usual +location for the local database. .SH OPTIONS .TP \-f @@ -48,7 +53,7 @@ and no files are changed, this can only detect and report some types of errors. .TP \-L -Don't check for library dependencies +Don't check for library dependencies, passed on to zpm-search. .TP \-A run zpm-syncfs in accept mode @@ -83,6 +88,7 @@ ZPMDB Nathan Wagner .SH SEE ALSO .BR zpm (8) +.BR zpm-search (8) .BR zpm-script (8) .BR zpm-repo (8) .BR zpm-syncfs (8) diff --git a/schema/age.sql b/schema/age.sql new file mode 100644 index 0000000..bdc5da8 --- /dev/null +++ b/schema/age.sql @@ -0,0 +1,24 @@ +create view package_age as +with aging as ( +select +P.package, P.version,P.release,P.status, +row_number() over ( + partition by P.package + order by P.version collate vercmp desc, P.release desc +) as age +from +packages P +left join preserve_packages KEEP +on KEEP.package = P.package and KEEP.version = P.version and KEEP.release = P.release +where P.status in ('updated','removed') +and KEEP.package is null +) +select * from aging +union +select P.package, P.version, P.release, P.status, 0 as age +from packages P where P.status = 'installed' +union +select P.package, P.version, P.release, 'preserved' as status, 0 as age +from preserve_packages P +order by package, version collate vercmp desc, release desc +; diff --git a/zpm-gc b/zpm-gc index 08ba1e1..d2d49fc 100755 --- a/zpm-gc +++ b/zpm-gc @@ -1,5 +1,6 @@ #!/bin/sh +set -e # garbage collect a package database # flags to skip phases @@ -20,6 +21,7 @@ remove_orphaned_elf=1 # -fF remove failed remove_failed=1 # -hH fix file content hash +remove_incomplete_packages=0 fix_hash=0 # -hH remove install history by date remove_history= @@ -27,29 +29,44 @@ remove_history= remove_logs= # -nN remove acked notes remove_acked_notes=0 -# -oO remove outdated packages, unless preserved -remove_old_packages=0 # -pP remove history for missing packages clean_missing_package_history=0 -# -rR remove orphaned rows in files table packagefiles, notes, scripts, ... -remove_orphaned_files=1 + # -hH remove all file content that is not scripts or configs remove_packagefile_content=1 remove_script_content=1 remove_config_content=1 # -rR remove removed packages, unless preserved remove_removed_packages=0 + # -vV compactify the database file -vacuum=1 +vacuum=0 + +# remove orphaned rows in files table packagefiles, notes, scripts, ... +unreferenced_file_content=1 + +# remove outdated packages, unless preserved +remove_old_packages=1 db=${ZPMDB:-/var/lib/zpm/local.db} -while getopts 'f:'; do +retain_old=${ZPM_KEEP_PACKAGES:-2} + +while getopts 'f:PcCk:' opt; do case $opt in f) db="$OPTARG" ;; + P) remove_old_packages=0 ;; + c) vacuum=1 ;; + C) unreferenced_file_content=0 ;; + k) retain_old="$OPTARG" ;; esac done +retain_old=$((retain_old + 0)) +if [ $retain_old -le 0 ]; then + retain_old=0 +fi + if [ -z "$db" ]; then echo must specify database file exit 1 @@ -65,10 +82,12 @@ export ZPMDB # remove failed packages for pkg in $(zpm list -s failed); do zpm rmpackage -m 'gc removed failed' "$pkg" + echo removed $pkg done for pkg in $(zpm list -s dryrun); do zpm rmpackage -m 'gc removed dryrun' "$pkg" + echo removed $pkg done # remove incomplete packages @@ -76,24 +95,27 @@ done if [ $remove_incomplete_packages -ne 0 ]; then for pkg in $(zpm list -F 'hash is null'); do zpm rmpackage -m 'gc removed incomplete' $pkg + echo removed $pkg done fi # remove old version packages, unless preserved -# TODO need 'rstatus' flag +# remove removed packages, unless preserved if [ $remove_old_packages -ne 0 ]; then - # TODO check for preserved - # TODO add ability to keep a certain number of back packages - zpm shell $ZPMDB "delete from packages where status = 'updated'" + zpm shell $ZPMDB "select printf('%s-%s-%s',package,version,release) from package_age where age > $retain_old" | while read pkg; do + zpm rmpackage -m 'gc removed old package' $pkg + echo removed $pkg +done fi -# remove removed packages, unless preserved - # remove orphaned rows in files table # references in packagefiles, notes, scripts, possibly others if [ $remove_orphaned_files -ne 0 ]; then - zpm shell $ZPMDB 'delete from files where hash in (select hash from filerefs where refcount = 0);' + count=$(zpm shell $ZPMDB 'delete from files where hash in (select hash from filerefs where refcount = 0);select changes()') + if [ $count -gt 0 ]; then + zpm log -v -i -a 'gc' -t $ZPMDB "removed $count unreferenced content" + fi fi # remove old logs, given by date @@ -107,17 +129,18 @@ fi # remove history for missing packages if [ $clean_missing_package_history -ne 0 ]; then - + # delete from zpmlog where ? + true fi # remove command history by date +# compactify the database file if [ $vacuum -ne 0 ]; then - # compactify the database file osize=$(zpm stat -f '%s' $ZPMDB) - zpm log -v -i -a 'gc compacting' -t $ZPMDB zpm shell $ZPMDB vacuum nsize=$(zpm stat -f '%s' $ZPMDB) + zpm log -v -i -a 'gc compacted' -t $ZPMDB "$osize -> $nsize" printf 'original size %s\n' $osize printf 'new size %s\n' $nsize -- 2.40.0