From d3b90e6aa8ab580c77ead76ff5c6c6dca7991cc6 Mon Sep 17 00:00:00 2001 From: Nathan Wagner Date: Thu, 4 Jan 2018 10:21:43 -0600 Subject: [PATCH] integrate previous work --- Commands | 84 +++++++++++++++++++++- README | 184 +++++++++++++++++++++++++++++++++++++++++++++-- db.sql | 57 +++++++++++++-- t/addtopackage.t | 7 +- t/tap.sh | 10 +-- zpm-addtopackage | 55 ++++++++++---- zpm-install | 28 ++++++-- zpm-ipkgfile | 18 +++-- zpm-newpackage | 2 +- zpm-pkginfo | 100 +++++++++++++++++++++++--- 10 files changed, 493 insertions(+), 52 deletions(-) diff --git a/Commands b/Commands index 04c5b61..94dbda2 100644 --- a/Commands +++ b/Commands @@ -1,6 +1,70 @@ zpm: always a shell script, that way it can source config files and export environment variables for sub-programs +Good to have a program to check if a package exists. That is, would +zpm install do anything, at least notionally. + +Three basic modes + +1: working on the local database and manipulating packages +This is where 'install' 'update' 'remove' etc are done. +Additionally, this is where notes, bug reports, repo +control, and so forth are done. Since this is the most +common mode, these commands shouldn't need a prefix or +something saying we're dealing with the local package DB. + +2: working on a package file. +This is where building a new package, adding files, and otherwise +dealing with package metadata happens + +3: working on a repository of package files + +Since 2 and 3 are the less common operations, they can have +more verbose syntax, also, 2 "expects" a package file, so +require an option to force using the local DB, and 3 expects +needs repos, perhaps force not using local DB +Difference between working on a repository, and manipulating +repository info for the local or other DB +And 1 expects the local db, so option to specify package file. + +-f for package file? or -p? and for cat 2, take as argument if +not specified? perhaps an option to construct? probably +not, have a program to construct if needed. + +Need a way to specify a non-default local database location, +/var/lib/zpm/local.zpm by default + +For 2: zpm pkg +For 3: zpm repo + +Conceivably for 2: 'zpm -f ', since if it's not the +local package database, you'll have to specify the package file + +install: will search the repos for the most recent package. +of install a from a package file given with -f. + +so zpm install -f 'pkgfile' will install all packages in that file +zpm install -f pkgfile package will install just the named package +zpm install -f pkfile package package2 will installed both packages +zpm install -f pkgfile -V 1.2 will install version 1.2 of all packages +that have a version 1.2 +zpm install -f pkgfile foo-1.2 will install version 1.2 of package foo + +zpm install foo will install the most up to date foo from the +repos + +zpm install foo-1.2 install version 1.2. how to distinguish +between versions and package names with hyphens? Use @? + +Package filters: Version, release, name, min/max version, min/max release +Package tags? + +File filters: Get everything, except exclude by tag, which will +exclude every file with the tag. Can reverse sense, and get +nothing, except what's in tag. +tag will add a tag to the set, +-tag will remove it. 'tag' sets the tag list + + Environment Variables --------------------- @@ -10,15 +74,29 @@ ZPMDB: path to local database ZPMPACKAGE: package name ZPMPKGVER: package version ZPMPKGREL: package release +ZPMPKGFILE: package file + +Command Line Options +-------------------- + +Used by all programs, so can be passed to zpm + +-f package file Package Selector ---------------- -p -V -R -version and release are always the latest available if not specified -for new packages, version defaults to 1.0, release to 1, but -may also be taken from the environment +version and release are always the latest available if not specified for new +packages, version defaults to 1.0, release to 1, but may also be taken from the +environment + +If there is more than one package name in a database the shortest name is the +default, and the earliest alphabetically after that. This makes it so that if +you have 'db' and 'db-dev' in the same package, db will be the default. Though +that's not the way to do it, instead you should tag files within a package, and +then filter the install. command line over-rides environment, which over-rides defaults diff --git a/README b/README index 4ae1e24..2a489f0 100644 --- a/README +++ b/README @@ -1,6 +1,53 @@ ZPM - the ultimate package manager ================================== +Like "git" for packages. + +Would like a zpm-status and zpm-log, which should do things like +list incompletely installed packages, perhaps unacknowledged notes, +and so forth. zpm log should mainly show install/upgrade/deletes, +and repository actions. + +Any machine readable outputs or inputs should be json, or something +easily parsable by shell and awk + +Specifying the Package Database +------------------------------- + +-f /path/to/db/or/file +env ZPMFILE only for single package ops +env ZPMDB +sourced in /etc/zpm.conf by setting environment variable? +default to /var/lib/zpm/local.zpm + +Should probably distinguish here between the "package" and the "localdb". +They're not different technically, but +thus zpm install -f /path will still record in the ZPMDB + +Specifying a Package Within a DB +-------------------------------- + +Need a name,version,release triple + +But any string can be used, the system will attempt to disambiguate. +Can use zpm-findpkg to show how it would be interpreted. + +Otherwise drawn from (first wins): +-p +env ZPMPKG + +If you need specific versions: +env ZPMPKGVER +env ZPMPKGREL + +Or use zpm-findpkg -a to find all packages that match + +and any of those can be embedded in the them. +but start with either full triple in either ZPMPKG or -p, +or implied if neither is set +zpm-findpkg will return a triple, so can always run +zpm-findpkg and + Features -------- @@ -9,14 +56,14 @@ binary installs package metadata package integrity - gpg signatures + gpg signatures :- use own signatures, or figure out PD way to get gpg sha256 sums - sha3 sums + sha3 sums - not implemented package contents integrity sha256 sums of installed files - sha3 sums - sha1 sums + sha3 sums - not implemented + sha1 sums - not implemented lists of directory contents package database editing @@ -88,6 +135,27 @@ Extract file content from a package zpm extract +Remote Repositories +------------------- + + # add a new remote, -n don't do a pull immediately + zpm remote add [-n] + + # show all the remotes + zpm remote list + # set the preference level, higher is better + # perhaps remote "order" with lower better + # alpha by name on ties + zpm remote preference + + zpm remote pull [name ...] # all if no name + zpm remote drop + zpm remote freeze # prevent updates? + # fetch content of remotes, cached in filesystem? + zpm remote fetch [-a] [package ...] + + # list + Packaging Software ------------------ @@ -145,3 +213,111 @@ Get the unix timestamp of a file Get the uid of a file zpm stat -f '%u' + +Repositories +------------ + +A repository is just an archive of packages, which amounts to package metadata, +plus possibly file path and hash information. That is, the packages and +packagefiles tables, and possibly the elf information. Could be pretty much +everything except the actual file content. Could put a repo column on all of +those tables, with NULL being no repo information. Otherwise separate tables. +For the file paths, or at least the repo, need a url, could require it to be +automatically constructed from the repo url, possibly with escapes. + +Need a repo table with the remote repo name (locally decided), remote repo url, +and perhaps preference and trust policy. + +Something like + + zpm repo add # repo info download/fetch url + zpm repo sync [name] # all if no name + zpm repo preference [name ...] + zpm repo clone # download all packages to some destination + +This might mean that the package identifier tuple would be the repo, name, version, release. Which is getting a bit complex, so we need some way to +set sensible defaults. + +Build Scripts +------------- + +A building a package with zpm build. + +Source a build script named ZBUILD, or package.build. Not sure which. +Either way, build script itself shouldn't do anything other +than setting variables and defining functions. + +zpm build variables: + +srcdir: where the source code is unpacked + +variables: + +source: a list of urls, suitable for curl to download +checksum: a corresponding list of sha256 checksums +PACKAGE: the name of the package +VERSION: the package version + +functions: + +install: should install the package under $pkgtree +can create more than one package under $pkgbase + +postpackage: optional a function to run after the package file is created, +one arg, path to zpm file + +build: compile the package + +internal zpm functions: + +Package File tags +----------------- + +There are no "subpackages", but you can tag paths +at install time you can include or exclude paths +you get ((tag match include) and (tag not match exclude)) +default include is all, and default exclude is none + +might use ^: for standard "what is this" tags, +and ^@ for "sub-package" tagging, or what amount +to optional parts of an install + +A tag must be no whitespace and no shell metacharacters + + [a-zA-Z:=@.]+ + +Hmm, it would make sense to tag regardless of package. +perhaps require -A or somesuch. + +setting tags when building a package: + + zpm tag [-r] -f $pkgfile tag [...] + +add tags + + zpm tag -a -f $pkgfile tag [...] + +remove tags + + zpm tag -d -f $pkgfile tag [...] + +list tags + + zpm tag -l -f $pkgfile + +test tag: i.e. does a file have (all) given tag(s) + + zpm tag -t -f $pkgfile tag [...] + +Standard tags +------------- + +dev: headers and static libraries +doc: man pages and other documentation +man: man pages +lib: libraries +shlib: shared libraries + +maybe prefix standard tags with ':' and then anything else +is free to be used. need a way to list all package tags +zpm pkgtags ? diff --git a/db.sql b/db.sql index 75e5354..0bdc654 100644 --- a/db.sql +++ b/db.sql @@ -8,9 +8,9 @@ PRAGMA user_version = 1; -- TODO copyright and license information should probably -- go here CREATE TABLE files ( - hash text primary key, - size integer, - compression text, + hash text primary key, -- sha256 of content + size integer, -- bigint? certainly need > 2GB + compression text, -- always xz? content blob ) ; @@ -22,6 +22,7 @@ create table packages ( package text, version text, -- the upstream version string release integer, -- the local release number + pkgid text, -- the three above joined with '-' -- metadata columns description text, @@ -37,8 +38,16 @@ create table packages ( without rowid ; +-- packagefile hash is columns as text, joined with null bytes, then +-- sha256 sum of that +-- package checksum is package columns as text, joined with null bytes, +-- other than the checksum and install_time column +-- then that hashed. finally, that hash, plus the ascii sorted +-- hashes of the package files all joined with newlines, hashed. +-- really don't like this. + -- files contained in a package -create table packagefiles ( +create table paths ( -- package id triple package text, version text, @@ -70,7 +79,7 @@ create table packagefiles ( without rowid ; -create table packagefiletags ( +create table pathtags ( -- package id triple package text, version text, @@ -148,4 +157,42 @@ create table packagegroups ( "group" text ); +-- zpm actions +-- not sure how machine readable this needs to be, +-- do not at all for now, figure it out later +-- could be worth logging all commands in a history table, +-- the zpm driver could do that and capture the exit status +-- as well +-- might want the history table to note a "group" to tie together +-- sub-invocations, probably an environment variable set if not +-- already set by zpm, probably a uuid or a timestamp +create table zpmlog ( + ts integer, -- timestamp of action, may need sub-second + action text, + target text, -- packagename, repo name, etc + info text -- human readable +); + +create table history ( + ts integer, -- again, probably needs timestamp sub second + cmd text, + args text, + status integer +); + +create table repository ( + name text primary key, -- our name for a repo + url text not null, + priority integer not null default 1, + refreshed integer -- last refresh time +); + +-- track which repository a package was cloned from, i.e. where we got it +create table packagesource ( + name text, + version text, + release integer, + repository text references repository +); + commit; diff --git a/t/addtopackage.t b/t/addtopackage.t index 3252bec..7fcb91f 100755 --- a/t/addtopackage.t +++ b/t/addtopackage.t @@ -7,7 +7,7 @@ PF=test.db -plan 15 +plan 17 td=test.addtopackage rm -rf $td @@ -47,6 +47,11 @@ okstreq "$3" "1" package release okstreq "$4" "$td/foo" file foo in package diag "$pkglist" +rm -f $PF +require zpm addtopackage -f $PF -S "$td" zpmtest $td/foo +fn=$(zpm showpkg $PF | awk '{print $4}') +okstreq "$fn" "foo" file foo in package prefix striped + finish rm -rf $td diff --git a/t/tap.sh b/t/tap.sh index b82f203..1811f5d 100755 --- a/t/tap.sh +++ b/t/tap.sh @@ -7,7 +7,7 @@ tryrun() { note="$@" program=$1 shift - $program "$@" + $program "$@" >> test.out 2>&1 if [ $? -ne 0 ]; then printf 'not '; fi @@ -19,9 +19,11 @@ require() { note="$@" program=$1 shift - $program "$@" - if [ $? -ne 0 ]; then - printf "bail out! fail[$?]: $@\n" + $program "$@" >> test.out 2>&1 + rv=$? + if [ $rv -ne 0 ]; then + diag "bailing on $((tn + 1)) $program $*" + printf "bail out! fail[$rv]: $@\n" exit 255; fi tn=$((tn + 1)) diff --git a/zpm-addtopackage b/zpm-addtopackage index a3b045b..9c66169 100755 --- a/zpm-addtopackage +++ b/zpm-addtopackage @@ -3,10 +3,33 @@ pkgver=${ZPMPKGVER:-1.0} pkgrel=${ZPMPKGREL:-1} +die() { + echo $* 1>&2 + exit 1 +} + +# basic cleanup on a path +cleanpath() { + clean="$1" + if [ -z "$clean" ]; then printf ''; fi + + # multiple slashes + clean=$(printf "%s" "$clean" | sed -e 's|/+|/|g') + # curdir + clean=$(printf "%s" "$clean" | sed -e 's|/\./|/|g') + # leading curdir + clean=${clean#./} + # trailing curdir + clean=${clean%/.} + # trailing slash + clean=${clean%/} + printf "%s" "$clean" +} + # option for "multipackage" just to let the system know that's what you meant # option to take filenames from stdin # parse package, version, release from file if not given -while getopts :f:v:r:d:a:u:l:p:b:P: opt; do +while getopts :f:v:r:d:a:u:l:p:b:P:S: opt; do case $opt in f) pkgfile="$OPTARG" ;; v) pkgver="$OPTARG" ;; @@ -18,17 +41,13 @@ while getopts :f:v:r:d:a:u:l:p:b:P: opt; do p) packager="$OPTARG" ;; b) builddate="$OPTARG" ;; P) prefix="$OPTARG" ;; + S) strip=$(cleanpath "$OPTARG"); ;; t) tags="$OPTARG" ;; c) tags="$tags +configuration" ;; esac done shift $((OPTIND - 1)) -die() { - echo $* 1&>2 - exit 1 -} - package="$1" shift if [ -z "$package" ]; then @@ -45,6 +64,7 @@ zpm newpackage -I -f $pkgfile -v $pkgver -r $pkgrel $package || exit 1 zpm test -v $pkgfile +#strip=$(cleanpath "$strip") for path in $*; do mtime=$(zpm stat -f '%y' $path) uid=$(zpm stat -f '%u' $path) @@ -52,20 +72,29 @@ for path in $*; do username=$(zpm stat -f '%U' $path) groupname=$(zpm stat -f '%G' $path) mode=$(zpm stat -f '%a' $path) + rpath="$path" + + rpath=$(cleanpath "$path") - # strip off leading slashes - rpath=$(echo "$path" | sed -e 's|^/*||') - # and a leading ./ - rpath=${rpath#./} - rpath=$(echo "$rpath" | sed -e 's|^/*||') + # strip off leading slash + rpath=${rpath#/} if [ -z "$rpath" ] || [ "$rpath" = '.' ]; then continue fi + if [ ! -z "$strip" ]; then + echo "stripping $strip" + rpath=${rpath#$strip} + rpath=${rpath#/} + fi + + if [ -z "$rpath" ]; then + die "$path resolves to nothing" + fi + + prefix=$(cleanpath "$prefix") if [ ! -z "$prefix" ]; then - # trailing slashes on prefix - prefix=$(echo "$prefix" | sed -e 's|/*$||') rpath="$prefix/$rpath" fi diff --git a/zpm-install b/zpm-install index 487740d..8d259c0 100755 --- a/zpm-install +++ b/zpm-install @@ -7,6 +7,8 @@ pkgrel=${ZPMPACKAGEREL:-1} pkgroot=/ # allocate an install id, perhaps hash package and timestamp +# installid=$(echo $(date) $pkglist | zpm hash) + # extract pre-scripts and run them # get list of paths to install # for each path, if directory, create and set mode @@ -18,9 +20,17 @@ pkgroot=/ # move into place # after all the files, extract post scripts and run them +# also need to mark package as installing so if it fails part way +# through, it can be finished later +# probably also want an option to "backup" any packages being upgraded +# so it's trivial to downgrade or to revert if the install fails + # option for "multipackage" just to let the system know that's what you meant # option to take filenames from stdin # parse package, version, release from file if not given +# TODO what's the difference between prefix and pkgroot +# need an option to not chown the files +# option to install but not merge/track while getopts :f:v:r:d:a:u:l:p:b:P:R: opt; do case $opt in R) pkgroot="$OPTARG" ;; @@ -38,20 +48,23 @@ while getopts :f:v:r:d:a:u:l:p:b:P:R: opt; do done shift $((OPTIND - 1)) -set -e +die() { + echo $* 1>&2 + exit 1 +} + if [ -z "$pkgfile" ]; then # actually, if no pkgfile, get pkgfile from repo + # but need to loop over finding package files then + # so this program probably needs to be "install from pkgfile" + # and a separate one that will loop over a package + # spec list and find from repos + die "must specify package file" pkgfile="$package-$pkgver-$pkgrel.zpm" fi -die() { - echo $* 1&>2 - exit 1 -} - set -e zpm test -v $pkgfile -set +e if [ $# -gt 0 ]; then pkglist="$@" @@ -75,6 +88,7 @@ for pkg in $pkglist; do # add package info to local package db # zpm merge -L -f $pkgfile $pkg + # check for conflicts # mark package in localdb as installing # zpm setmark installing $pkg # install all the files for a package diff --git a/zpm-ipkgfile b/zpm-ipkgfile index 310a7b7..04f980e 100755 --- a/zpm-ipkgfile +++ b/zpm-ipkgfile @@ -20,7 +20,7 @@ done shift $((OPTIND - 1)) die() { - echo $* 1&>2 + echo $* 1>&2 exit 1 } @@ -60,11 +60,21 @@ for path in $*; do tmppath=$dir/.installing.$name - cat <<-EOC + # can't use install because it's not posix. + # probably worth writing a zpm-install, but with + # a different name since that wants to be a package install + # probably makes the most sense for extract to know how + # to do the extract to temp and such, then + # you can just add the -S option (secure) + #cat <<-EOC + set -x mkdir -p $dir zpm extract $pkgfile $fhash $tmppath 0 chown $owner:$group $tmppath chmod $mode $tmppath - mv $tmppath $dir/$name - EOC + # TODO mv -n non-posix, going to need to have extract do all + # of this + mv -n $tmppath $dir/$name + set +x + #EOC done diff --git a/zpm-newpackage b/zpm-newpackage index 8cce97d..3afabb6 100755 --- a/zpm-newpackage +++ b/zpm-newpackage @@ -22,7 +22,7 @@ done shift $(( OPTIND - 1)) die() { - echo $* 1&>2 + echo $* 1>&2 exit 1 } diff --git a/zpm-pkginfo b/zpm-pkginfo index 93326e5..bf735c7 100755 --- a/zpm-pkginfo +++ b/zpm-pkginfo @@ -1,7 +1,6 @@ #!/bin/sh package=${1:-$ZPMPACKAGE} -shift pkgver=${ZPMPACKAGEVER:-1.0} pkgrel=${ZPMPACKAGEREL:-1} @@ -27,23 +26,104 @@ while getopts :f:v:r:d:a:u:l:p:b:P: opt; do esac done +pkgfile=$1 + set -e if [ -z "$pkgfile" ]; then pkgfile="$package-$pkgver-$pkgrel.zpm" fi -appid=$(sqlite3 $pkgfile 'pragma application_id;' | ( echo obase = 16; cat - ) | bc) -if [ "$appid" != "5A504442" ]; then - echo $pkgfile does not appear to be a zpm package file - exit 1 -fi +zpm-test -v $pkgfile +pkg=$(zpm-findpkg $pkgfile) +#.mode line { -sqlite3 $pkgfile <