From ecdd57da7df775c039aee834e1f74172004f352b Mon Sep 17 00:00:00 2001 From: Nathan Wagner Date: Fri, 16 Sep 2016 23:52:48 +0000 Subject: [PATCH] large commit of C work --- .gitignore | 9 + Commands | 32 +++ DATABASE | 140 ++++++++++++ Makefile | 27 ++- README | 72 +++++++ compress.c | 102 +++++++++ db.sql | 108 ++++++++++ lib/compress.c | 7 +- lib/db.c | 70 ++++++ lib/sha256.h | 213 ++++++++++++++++++ lib/uncompress.c | 104 +++++++++ lib/zpm.c | 539 ++++++++++++++++++++++++++++++++++++++++++++++ uncompress.c | 198 +++++++++++++++++ zpm-addfile.c | 20 ++ zpm-addtopackage | 96 +++++++++ zpm-extract.c | 118 ++++++++++ zpm-extractfile.c | 381 ++++++++++++++++++++++++++++++++ zpm-fileinfo | 49 +++++ zpm-getlibs | 55 +++++ zpm-install | 102 +++++++++ zpm-ipkgfile | 10 + zpm-newpackage | 37 ++++ zpm-pkg | 43 ++++ zpm-pkginfo | 49 +++++ zpm-preserve | 43 +++- zpm-showpkg | 13 ++ zpm.h | 93 ++++++++ 27 files changed, 2715 insertions(+), 15 deletions(-) create mode 100644 Commands create mode 100644 DATABASE create mode 100644 README create mode 100644 compress.c create mode 100644 db.sql create mode 100644 lib/db.c create mode 100644 lib/sha256.h create mode 100644 lib/uncompress.c create mode 100644 lib/zpm.c create mode 100644 uncompress.c create mode 100644 zpm-addfile.c create mode 100755 zpm-addtopackage create mode 100644 zpm-extract.c create mode 100644 zpm-extractfile.c create mode 100755 zpm-fileinfo create mode 100755 zpm-getlibs create mode 100755 zpm-install create mode 100755 zpm-ipkgfile create mode 100755 zpm-newpackage create mode 100755 zpm-pkg create mode 100755 zpm-pkginfo create mode 100755 zpm-showpkg create mode 100644 zpm.h diff --git a/.gitignore b/.gitignore index 941a639..ecc5e6a 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,10 @@ elftype +*.o +*.a +zpm-addfile +zpm-extract +uncompress +*.zpm +*.db +soname +compress diff --git a/Commands b/Commands new file mode 100644 index 0000000..82049d4 --- /dev/null +++ b/Commands @@ -0,0 +1,32 @@ +note: + add a note file, - from stdin, list if none + +edit: + edit a note file + +ack: + acknowledge a note file + +build: + build a package from source + - take file names on stdin and build package from them + - repackage an installed package + +install: + install a package, - for from a file + +remove: + remove a package + +db: + edit package databases used for finding packages + +info: + get information on a package + +track: + add a package to config tracked packages + +add: add to a repository/database + +clean: clean a repository/database diff --git a/DATABASE b/DATABASE new file mode 100644 index 0000000..d54da7b --- /dev/null +++ b/DATABASE @@ -0,0 +1,140 @@ +Packages are sqlite databases + +get application id and userver + +Primitive operations: + +add blob to repo +add path to repo +associate path with package +associate blob with path? +extract blob to a path +compare blob to filesystem path +create package with info + +Extra primitives: + +record elf information about blob +compress blob +uncompress blob +sign a package? What are we verifying? + +Signatures +---------- + +things: + +files, just a table of file contents, hash to content, +avoids redundancy. possibly type? size? internal compression? +blobs might make more sense as a name. + +package files: table of pathnames in a package, with metadata +file hash, type (directory, file, symlink, etc), permissions, +owner, group, acls?, config file, and package name... + +create table paths ( + path text, + mode integer, -- perms, use text for octal rep? + hash text, -- what should go here, null for dir + mtime integer -- seconds since epoch, but allow finer? +); + +create table packagefiles ( + package text, + subpackage text, -- libs, dev, client, server, whatever + path text, + filetype text -- e.g. config, etc? +); + +-- TODO just elf information? +create table libraries ( + package text, + subpackage text, + path text, + soname text +); + +create table librarydeps ( + package text, + subpackage text, + path text, + soname text -- soname of dependency +); + +-- package scripts: table of package, stage, file +create table scripts ( + package text, + subpackage text, + hash text +); + +-- package dependencies: table of package, dependency, dep type (package, soname) +create table packagedeps ( + package text, + subpackage text, + requires text, -- package name + subreq text, -- if requires only a sub package, probably most common for libs + minversion text, + maxversion text +); + +-- capability labels +create table provides ( + package text, + subpackage text, + label text -- a capability label +); + +create table requires ( + package text, + subpackage text, + label text -- a capability label +); + +create table packages ( + package text, + version text, -- the upstream version string + release integer, -- the local release number + description text, + architecture text, + url text, + licenses text, -- hash of actual license? need table for more than one? + packager text, + build_date integer, + install_date integer +); + +create table packagegroups ( + package text, + group text +); + +packages: table of package info +name, version, release, +tied package? i.e. another package/version/release this one goes with. + +Actual Package Install DB: +-------------------------- + +packages table: +package, name, version, release, install ts, uninstall/upgrade ts + +contents table: +from package files table in package, plus install info + +Installing a Package +-------------------- + +insert into DB.packages the package info from the .zpkg file, leave install date null +extract the pre-install script and run it. abort if exit failure +insert into the contents table info from the ZP.packagefiles table +for each file in the new contents, extract the file to a temporary location +in the same directory, rename into place, update as installed +for each file in the old contents, remove the file if it's not in the new contents +delete the row from the contents table +update the uninstall and install ts as a single transaction +extract and run post-install script + +pre/post install scripts get arguments: +package name, new version, new release, old version, old release +old version/release only if upgrade. diff --git a/Makefile b/Makefile index 239047f..053af31 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,12 @@ -CFLAGS=-Wall -std=c99 +CFLAGS=-Wall -std=c99 -I. -L. +LDFLAGS=-L. -programs: elftype soname +def: libzpm.a + +programs: elftype soname zpm-addfile zpm-extract + +uncompress: uncompress.o + $(CC) $(CFLAGS) -o $@ $+ -llzma -lsqlite3 elftype: elf/elftype.c $(CC) $(CFLAGS) -o $@ $+ @@ -8,6 +14,20 @@ elftype: elf/elftype.c soname: elf/soname.c $(CC) $(CFLAGS) -o $@ $+ +zpm-addfile: zpm-addfile.o libzpm.a + $(CC) $(CFLAGS) -o $@ $< -lsqlite3 -llzma -lzpm + +zpm-extract: zpm-extract.o libzpm.a + $(CC) $(CFLAGS) -o $@ $< -lsqlite3 -llzma -lzpm + +newdb.c: db.sql + echo "char createdb[] = {" > $@ + xxd -i < $< >> $@ + echo "};" >> $@ + +libzpm.a: lib/sha256.o lib/db.o lib/compress.o lib/uncompress.o newdb.o lib/zpm.o + ar rcuv $@ $+ + install: elftype install -D zpm $(DESTDIR)/bin/zpm install -D zpm-note $(DESTDIR)/bin/zpm-note @@ -21,3 +41,6 @@ install: elftype install -D elftype $(DESTDIR)/usr/sbin/elftype install -D soname $(DESTDIR)/usr/sbin/soname #SPOOL=$(DESTDIR)/var/lib/admin/notes ./zpm-sequence -c notes + +clean: + rm -f *.o diff --git a/README b/README new file mode 100644 index 0000000..e58cc90 --- /dev/null +++ b/README @@ -0,0 +1,72 @@ +Features +-------- + +binary installs + +package metadata + +package integrity + gpg signatures + sha256 sums + sha3 sums + +package contents integrity + sha256 sums of installed files + sha3 sums + sha1 sums + lists of directory contents + +package database editing + e.g. marking a file as a configuration file + +repackaging + creating a package from the current state of an installed package + +package a tarball + create a package from a tarball + +package from stdin + like cpio + +package a directory + like tar + +build from shell scripts + +rebuild a package + +write ahead log for package filesystem operations + it should always be possible for the package system + to figure out if it was interrupted + +install, uninstall, upgrade, and downgrade hooks + +dependency tracking + +network fetching of packages and package databases +reference counting of dynamic libs + +simple packaging constructs + use shell scripts where possible + use shell functions as hooks and callbacks + +package is just a tarball +can contain more than one package +xz compressed + +version? +package-name/version/files/{usr/bin/foo, etc} +package-name/version/info/hooks - shell script of functions +package-name/version/info/pkginfo - shell readable +package-name/version/info/manifest? derivable from tarball +package-name/version/info/dynamiclibs, or greppable from manifest? + +/var/lib/zpac: +packages/installed//files/sha1/ab/ab/hash -> file +librefs// (or file?) +sha1/ab/ab/ -> file +fileowner/sha1/ab/ab/hash/packagename - can be more than one + +need to mark a package as held for no automatic upgrade + +configure to run a command after an any package work diff --git a/compress.c b/compress.c new file mode 100644 index 0000000..648a6d9 --- /dev/null +++ b/compress.c @@ -0,0 +1,102 @@ +/* + * add a file/files to an sqlite db + * in the 'files' table. + */ + +#include +#include +#include +#include +#include + +#include + +#include "lzma.h" + +#include "sha256.h" + +void *compresslzma(void *buf, size_t bufsize, size_t *len) { + /* xz compress it */ + size_t outsize; + void *outbuf; + size_t outlen = 0; + + outsize = lzma_stream_buffer_bound(bufsize); + outbuf = malloc(outsize); + /* TODO adjust encoding level for size */ + lzma_easy_buffer_encode(6, LZMA_CHECK_CRC64, NULL, + buf, bufsize, + outbuf, &outlen, outsize + ); + *len = outlen; + return outbuf; +} + +int main(int ac, char **av){ + int i, j; + + /* insert each file */ + for (i = 1; i < ac; i++) { + int fd; + void *content; + struct stat sbuf; + unsigned char tmp[32]; + char hash[65]; /* hash in hex */ + hash_state md; + + fd = open(av[i], O_RDONLY); + if (fd == -1) { + exit(1); + } + if (fstat(fd, &sbuf) == -1) { + exit(1); + } + /* not a regular file? */ + if (!S_ISREG(sbuf.st_mode)) { + exit(1); + } + + content = mmap(0, sbuf.st_size, PROT_READ,MAP_PRIVATE, fd, 0); + if (!content) { + exit(2); + } + + sha256_init(&md); + sha256_process(&md, content, sbuf.st_size); + sha256_done(&md, tmp); + for (j=0;j<32;j++) { + sprintf(hash+j*2, "%02x", (unsigned)tmp[j]); + } + hash[64] = 0; + fprintf(stderr, "file %s: %s\n", av[i], hash); + + /* xz compress it */ + size_t outlen = 0; + FILE *save; + void *outbuf; + + outbuf = compresslzma(content, sbuf.st_size, &outlen); + save = fopen("foo.xz", "w"); + fwrite(outbuf, outlen, 1, save); + fclose(save); + free(outbuf); + + //Find out the maximum size of the compressed output for single-call compression: +// zlib: compressBound() +// liblzma: lzma_stream_buffer_bound() + +// Compressing: +//zlib: compress() or compress2() + // liblzma: lzma_easy_buffer_encode() or lzma_stream_buffer_encode() + + // Decompressing: +//zlib: uncompress() + // liblzma: lzma_stream_buffer_decode() + + // See container.h (src/liblzma/api/lzma/container.h in the source tree) for details how to use these functions. + + /* either way we're done with this now */ + munmap(content, sbuf.st_size); + } + return 0; +} diff --git a/db.sql b/db.sql new file mode 100644 index 0000000..7ad22b9 --- /dev/null +++ b/db.sql @@ -0,0 +1,108 @@ +begin; + +PRAGMA application_id = 0x5a504442; +PRAGMA user_version = 1; + +-- should be faster with rowid due to the blob content +CREATE TABLE files ( + hash text primary key, + size integer, + compression text, + content blob +) +; + +create table packages ( + package text, + version text, -- the upstream version string + release integer, -- the local release number + description text, + architecture text, + url text, + licenses text, -- hash of actual license? need table for more than one? + packager text, + build_time integer default (strftime('%s', 'now')), + install_time integer, + primary key (package,version,release) +) +without rowid +; + +create table packagefiles ( + package text, + version text, + release integer, + path text, + mode text, -- perms, use text for octal rep? + username text, -- name of owner + groupname text, -- group of owner + --filetype integer default 0, -- 0 regular file, 1 directory, 2 symlink + -- regular file if null target and not null hash + -- except that we could not know the hash, or care + -- directory if null hash and null target + -- symlink if null hash and not null target + -- hard link if not null hash and not null target + -- device special files add dev number column + -- fifos add mode? Can encode filetype in mode. + target text, -- link target + hash text, -- what should go here, null for dir? + mtime integer, -- seconds since epoch, but allow finer? + primary key (package,version,release,path), + foreign key (package,version,release) references packages (package,version,release) on delete cascade +) +without rowid +; + +-- TODO just elf information? +-- and just hash, not package? +create table libraries ( + package text, + subpackage text, + path text, + soname text +) +; + +create table librarydeps ( + package text, + subpackage text, + path text, + soname text -- soname of dependency +); + +-- package scripts: table of package, stage, file +create table scripts ( + package text, + subpackage text, + stage text, + hash text +); + +-- package dependencies: table of package, dependency, dep type (package, soname) +create table packagedeps ( + package text, + subpackage text, + requires text, -- package name + subreq text, -- if requires only a sub package, probably most common for libs + minversion text, + maxversion text +); + +-- capability labels +create table provides ( + package text, + subpackage text, + label text -- a capability label +); + +create table requires ( + package text, + subpackage text, + label text -- a capability label +); + +create table packagegroups ( + package text, + "group" text +); +commit; diff --git a/lib/compress.c b/lib/compress.c index a7427cd..5101688 100644 --- a/lib/compress.c +++ b/lib/compress.c @@ -1,8 +1,3 @@ -/* - * add a file/files to an sqlite db - * in the 'files' table. - */ - #include #include #include @@ -16,6 +11,8 @@ #include "lzma.h" +/* TODO pass in the outbuf ? */ +/* or wrap lzma_stream_buffer_bound */ void *compresslzma(void *buf, size_t bufsize, size_t *len) { /* xz compress it */ size_t outsize; diff --git a/lib/db.c b/lib/db.c new file mode 100644 index 0000000..e09732e --- /dev/null +++ b/lib/db.c @@ -0,0 +1,70 @@ +/* + * add a file/files to an sqlite db + * in the 'files' table. + */ + +#include +#include +#include +#include +#include + +#include + +#include +#include "sha256.h" + + +static int callback(void *NotUsed, int argc, char **argv, char **azColName){ + int i; + for(i=0; i +#include +#include +#include +#include +#include +#include + +#include + +/* max size of either a cipher/hash block or symmetric key [largest of the two] */ +#define MAXBLOCKSIZE 128 + +/* descriptor table size */ +#define TAB_SIZE 32 + +/* error codes [will be expanded in future releases] */ +enum { + CRYPT_OK=0, /* Result OK */ + CRYPT_ERROR, /* Generic Error */ + CRYPT_NOP, /* Not a failure but no operation was performed */ + + CRYPT_INVALID_KEYSIZE, /* Invalid key size given */ + CRYPT_INVALID_ROUNDS, /* Invalid number of rounds */ + CRYPT_FAIL_TESTVECTOR, /* Algorithm failed test vectors */ + + CRYPT_BUFFER_OVERFLOW, /* Not enough space for output */ + CRYPT_INVALID_PACKET, /* Invalid input packet given */ + + CRYPT_INVALID_PRNGSIZE, /* Invalid number of bits for a PRNG */ + CRYPT_ERROR_READPRNG, /* Could not read enough from PRNG */ + + CRYPT_INVALID_CIPHER, /* Invalid cipher specified */ + CRYPT_INVALID_HASH, /* Invalid hash specified */ + CRYPT_INVALID_PRNG, /* Invalid PRNG specified */ + + CRYPT_MEM, /* Out of memory */ + + CRYPT_PK_TYPE_MISMATCH, /* Not equivalent types of PK keys */ + CRYPT_PK_NOT_PRIVATE, /* Requires a private PK key */ + + CRYPT_INVALID_ARG, /* Generic invalid argument */ + CRYPT_FILE_NOTFOUND, /* File Not Found */ + + CRYPT_PK_INVALID_TYPE, /* Invalid type of PK key */ + CRYPT_PK_INVALID_SYSTEM,/* Invalid PK system specified */ + CRYPT_PK_DUP, /* Duplicate key already in key ring */ + CRYPT_PK_NOT_FOUND, /* Key not found in keyring */ + CRYPT_PK_INVALID_SIZE, /* Invalid size input for PK parameters */ + + CRYPT_INVALID_PRIME_SIZE,/* Invalid size of prime requested */ + CRYPT_PK_INVALID_PADDING /* Invalid padding on input */ +}; + +#define LTC_MUTEX_GLOBAL(x) +#define LTC_MUTEX_PROTO(x) +#define LTC_MUTEX_TYPE(x) +#define LTC_MUTEX_INIT(x) +#define LTC_MUTEX_LOCK(x) +#define LTC_MUTEX_UNLOCK(x) + +#define STORE32H(x, y) \ + { (y)[0] = (unsigned char)(((x)>>24)&255); (y)[1] = (unsigned char)(((x)>>16)&255); \ + (y)[2] = (unsigned char)(((x)>>8)&255); (y)[3] = (unsigned char)((x)&255); } + +#define LOAD32H(x, y) \ + { x = ((unsigned long)((y)[0] & 255)<<24) | \ + ((unsigned long)((y)[1] & 255)<<16) | \ + ((unsigned long)((y)[2] & 255)<<8) | \ + ((unsigned long)((y)[3] & 255)); } + +#define STORE64H(x, y) \ + { (y)[0] = (unsigned char)(((x)>>56)&255); (y)[1] = (unsigned char)(((x)>>48)&255); \ + (y)[2] = (unsigned char)(((x)>>40)&255); (y)[3] = (unsigned char)(((x)>>32)&255); \ + (y)[4] = (unsigned char)(((x)>>24)&255); (y)[5] = (unsigned char)(((x)>>16)&255); \ + (y)[6] = (unsigned char)(((x)>>8)&255); (y)[7] = (unsigned char)((x)&255); } + +/* rotates the hard way */ +#define ROL(x, y) ( (((unsigned long)(x)<<(unsigned long)((y)&31)) | (((unsigned long)(x)&0xFFFFFFFFUL)>>(unsigned long)(32-((y)&31)))) & 0xFFFFFFFFUL) +#define ROR(x, y) ( ((((unsigned long)(x)&0xFFFFFFFFUL)>>(unsigned long)((y)&31)) | ((unsigned long)(x)<<(unsigned long)(32-((y)&31)))) & 0xFFFFFFFFUL) +#define ROLc(x, y) ( (((unsigned long)(x)<<(unsigned long)((y)&31)) | (((unsigned long)(x)&0xFFFFFFFFUL)>>(unsigned long)(32-((y)&31)))) & 0xFFFFFFFFUL) +#define RORc(x, y) ( ((((unsigned long)(x)&0xFFFFFFFFUL)>>(unsigned long)((y)&31)) | ((unsigned long)(x)<<(unsigned long)(32-((y)&31)))) & 0xFFFFFFFFUL) + + +#ifndef MIN + #define MIN(x, y) ( ((x)<(y))?(x):(y) ) +#endif + +struct sha256_state { + uint64_t length; + uint32_t state[8], curlen; + unsigned char buf[64]; +}; + +typedef union Hash_state { + char dummy[1]; + struct sha256_state sha256; + void *data; +} hash_state; + +/** hash descriptor */ +extern struct ltc_hash_descriptor { + /** name of hash */ + char *name; + /** internal ID */ + unsigned char ID; + /** Size of digest in octets */ + unsigned long hashsize; + /** Input block size in octets */ + unsigned long blocksize; + /** ASN.1 OID */ + unsigned long OID[16]; + /** Length of DER encoding */ + unsigned long OIDlen; + + /** Init a hash state + @param hash The hash to initialize + @return CRYPT_OK if successful + */ + int (*init)(hash_state *hash); + /** Process a block of data + @param hash The hash state + @param in The data to hash + @param inlen The length of the data (octets) + @return CRYPT_OK if successful + */ + int (*process)(hash_state *hash, const unsigned char *in, unsigned long inlen); + /** Produce the digest and store it + @param hash The hash state + @param out [out] The destination of the digest + @return CRYPT_OK if successful + */ + int (*done)(hash_state *hash, unsigned char *out); + /** Self-test + @return CRYPT_OK if successful, CRYPT_NOP if self-tests have been disabled + */ + int (*test)(void); + + /* accelerated hmac callback: if you need to-do multiple packets just use the generic hmac_memory and provide a hash callback */ + int (*hmac_block)(const unsigned char *key, unsigned long keylen, + const unsigned char *in, unsigned long inlen, + unsigned char *out, unsigned long *outlen); + +} hash_descriptor[]; + +int sha256_init(hash_state * md); +int sha256_process(hash_state * md, const unsigned char *in, unsigned long inlen); +int sha256_done(hash_state * md, unsigned char *hash); +int sha256_test(void); +extern const struct ltc_hash_descriptor sha256_desc; + +int find_hash(const char *name); +int find_hash_id(unsigned char ID); +int find_hash_oid(const unsigned long *ID, unsigned long IDlen); +int find_hash_any(const char *name, int digestlen); +int register_hash(const struct ltc_hash_descriptor *hash); +int unregister_hash(const struct ltc_hash_descriptor *hash); +int hash_is_valid(int idx); + +LTC_MUTEX_PROTO(ltc_hash_mutex) + +int hash_memory(int hash, + const unsigned char *in, unsigned long inlen, + unsigned char *out, unsigned long *outlen); +int hash_memory_multi(int hash, unsigned char *out, unsigned long *outlen, + const unsigned char *in, unsigned long inlen, ...); + +#ifndef LTC_NO_FILE +int hash_filehandle(int hash, FILE *in, unsigned char *out, unsigned long *outlen); +int hash_file(int hash, const char *fname, unsigned char *out, unsigned long *outlen); +#endif + +/* a simple macro for making hash "process" functions */ +#define HASH_PROCESS(func_name, compress_name, state_var, block_size) \ +int func_name (hash_state * md, const unsigned char *in, unsigned long inlen) \ +{ \ + unsigned long n; \ + int err; \ + LTC_ARGCHK(md != NULL); \ + LTC_ARGCHK(in != NULL); \ + if (md-> state_var .curlen > sizeof(md-> state_var .buf)) { \ + return CRYPT_INVALID_ARG; \ + } \ + while (inlen > 0) { \ + if (md-> state_var .curlen == 0 && inlen >= block_size) { \ + if ((err = compress_name (md, (unsigned char *)in)) != CRYPT_OK) { \ + return err; \ + } \ + md-> state_var .length += block_size * 8; \ + in += block_size; \ + inlen -= block_size; \ + } else { \ + n = MIN(inlen, (block_size - md-> state_var .curlen)); \ + memcpy(md-> state_var .buf + md-> state_var.curlen, in, (size_t)n); \ + md-> state_var .curlen += n; \ + in += n; \ + inlen -= n; \ + if (md-> state_var .curlen == block_size) { \ + if ((err = compress_name (md, md-> state_var .buf)) != CRYPT_OK) { \ + return err; \ + } \ + md-> state_var .length += 8*block_size; \ + md-> state_var .curlen = 0; \ + } \ + } \ + } \ + return CRYPT_OK; \ +} + +#endif +/* TOMCRYPT_H_ */ diff --git a/lib/uncompress.c b/lib/uncompress.c new file mode 100644 index 0000000..867e5bc --- /dev/null +++ b/lib/uncompress.c @@ -0,0 +1,104 @@ +#include +#include +#include +#include +#include +#include + +#include + +#include + +#include "sha256.h" + +#include "lzma.h" + +void uncompresslzma(void *buf, size_t bufsize, FILE *out) { + lzma_stream s = LZMA_STREAM_INIT; + lzma_stream *strm; + + uint8_t outbuf[BUFSIZ]; + + int ret; + + strm = &s; + + ret = lzma_stream_decoder(strm, UINT64_MAX, 0); + /* The only reasonable error here is LZMA_MEM_ERROR. */ + if (ret != LZMA_OK) { + fprintf(stderr, "%s", ret == LZMA_MEM_ERROR ? strerror(ENOMEM) + : "Internal error (bug)"); + exit(EXIT_FAILURE); + } + + strm->avail_in = bufsize; + strm->next_in = buf; + strm->avail_out = BUFSIZ; + strm->next_out = outbuf; + + lzma_action action = LZMA_RUN; + + while (1) { + ret = lzma_code(strm, action); + + // Write and check write error before checking decoder error. + // This way as much data as possible gets written to output + // even if decoder detected an error. + if (strm->avail_out == 0 || ret != LZMA_OK) { + const size_t write_size = BUFSIZ - strm->avail_out; + + if (fwrite(outbuf, 1, write_size, out) != write_size) { + // Wouldn't be a surprise if writing to stderr + // would fail too but at least try to show an + // error message. + fprintf(stderr, "Cannot write to output file stream: " + "%s", strerror(errno)); + exit(EXIT_FAILURE); + } + + strm->next_out = outbuf; + strm->avail_out = BUFSIZ; + } + + if (ret != LZMA_OK) { + if (ret == LZMA_STREAM_END) { + // lzma_stream_decoder() already guarantees + // that there's no trailing garbage. + assert(strm->avail_in == 0); + //assert(action == LZMA_FINISH); + return; + } + + const char *msg; + switch (ret) { + case LZMA_MEM_ERROR: + msg = strerror(ENOMEM); + break; + + case LZMA_FORMAT_ERROR: + msg = "File format not recognized"; + break; + + case LZMA_OPTIONS_ERROR: + // FIXME: Better message? + msg = "Unsupported compression options"; + break; + + case LZMA_DATA_ERROR: + msg = "File is corrupt"; + break; + + case LZMA_BUF_ERROR: + msg = "Unexpected end of input"; + break; + + default: + msg = "Internal error (bug)"; + break; + } + + fprintf(stderr, "xz: %s\n", msg); + exit(EXIT_FAILURE); + } + } +} diff --git a/lib/zpm.c b/lib/zpm.c new file mode 100644 index 0000000..37c7e7b --- /dev/null +++ b/lib/zpm.c @@ -0,0 +1,539 @@ +#include +#include +#include +#include +#include +#include +#include +#include + +#include "zpm.h" + +#include "sha256.h" +#if 0 +struct zpm { + sqlite3 *db; + char *path; /* path to package file */ + char *version; + int release; + char *pkgname; + time_t installed; /* install time, 0 for not installed */ +}; + +struct zpm_file { + char *path; + int mode; + uint32_t filetype; + char *tags; + char *owner; + char *group; + char *hash; /* could be fixed length */ + time_t mtime; + struct zpm_file *next; /* so you can make a linked list */ +}; + +/* NULL? Create? */ +/* associate with a package ? if only one? first? */ +int zpm_open(struct zpm *pkg, char *path); +int zpm_pkgname(char *base, char *version, int release); /* construct a package file name */ + +/* flags for preserving mode, owner, etc */ +/* puts hash of import in hash */ +/* path can be a hash, with an "INTERNAL" flag, i.e. internally import */ +#define ZPM_MODE 0x1 +#define ZPM_OWNER 0x2 +#define ZPM_MTIME 0x4 +#define ZPM_INTERNAL 0x8 +#define ZPM_NOBLOB 0x10 +/* don't run scripts on install */ +#define ZPM_NOSCRIPTS 0x10 +/* don't associate the file with a package, just do a raw insert */ +/* otherwise, associate it with the current package */ +#define ZPM_NOPACKAGE 0x20 + +int zpm_import(struct zpm *zp, char *path, uint32_t flags, uint8_t *hash); + +/* link and unlink hashes to packages */ +int zpm_link(struct zpm *pkg, char *path, char *hash, struct zpm_file *fileinfo); +int zpm_unlink(struct zpm *pkg, char *path); + +/* tag a file. relative to "current package" */ +int zpm_tag(struct zpm *zp, char *path, char *tags); +/* should this be broken up into separage functions ? */ +int zpm_md(struct zpm *zp, char *path, int mode, char *owner, char *group, time_t mtime); + +/* export hash to dest */ +int zpm_extract(struct zpm *pkg, char *hash, char *path, int mode); + +/* export path to dest */ +int zpm_export(struct zpm *zp, char *path, uint32_t flags, char *dest); + +int zpm_close(struct zpm *zp); + +/* attach a signature to a package */ +int zpm_sign(struct zpm *z, size_t s, void *signature); + +/* set the package info to the nth package, -1 to return count? */ +/* further import/exports and such will be relative to this package */ +int zpm_package(struct zpm *zp, int n); + +/* get file information */ +int zpm_stat(struct zpm *z, struct zpm_file *f, int n); + +/* will also set the package context to the new package */ +int zpm_newpkg(struct zpm *z, char *base, char *version, int release); + +/* transactions */ +int zpm_begin(struct zpm *z); +int zpm_commit(struct zpm *z); +int zpm_rollback(struct zpm *z); + +/* higher level operations */ + +/* install or uninstall the package */ +/* flag for keeping the blobs in local */ +/* what about tag filtering */ +int zpm_install(struct zpm *z, struct zpm *local, uint32_t flags); +int zpm_uninstall(struct zpm *local); + +/* slurp up the actual blobs */ +/* what about versioning them if they change */ +int zpm_preserve(struct zpm *local); + +/* check file integrity */ +int zpm_checkinstall(struct zpm *local); + +int zpm_merge(struct zpm *z, struct zpm *src, uint32_t flags); + +void uncompresslzma(void *buf, size_t bufsize, FILE *out); +#define SQLERROR(x) fprintf(stderr, "%s %d: %s\n", __func__, __LINE__, (x)) +#endif + +static char *dupstr(char *s) { + size_t n; + char *d; + + n = strlen(s); + d = malloc(n+1); + if (d) { + d = strcpy(d, s); + } + return d; +} + +#if 0 +int zpm_newpkg(struct zpm *z, char *base, char *version, int release) { + char *sql = "insert or ignore into packages (package,version,release) values (?,?,?)"; + int rc; + sqlite3_stmt *ifile; + + rc = sqlite3_prepare(db, sql, -1, &ifile,0); + if (rc != SQLITE_OK) { + SQLERROR(sqlite3_errmsg(db)); + return 0; + } + rc = sqlite3_bind_text(ifile, 1, base, strlen(base), SQLITE_STATIC); + if (rc != SQLITE_OK) { + SQLERROR(sqlite3_errmsg(db)); + fprintf(stderr, "cant bind package name\n"); + zpm_rollback(pkg); + return 0; + } + sqlite3_bind_text(ifile, 2, version, strlen(version), SQLITE_STATIC); + sqlite3_bind_int(ifile, 3, release) + + rc = sqlite3_step(ifile); + + if (rc != SQLITE_DONE) { + SQLERROR(sqlite3_errmsg(db)); + sqlite3_finalize(ifile); + return 0; + } + sqlite3_finalize(ifile); + z->pkg = dupstr(base); + z->version = dupstr(version); + z->release = release; +} +#endif + +int zpm_begin(struct zpm *z) { + char *errstr = 0; + sqlite3_exec(z->db, "begin;", NULL, NULL, &errstr); + if (errstr) { + fprintf(stderr, "sqlite begin error: %s\n", errstr); + sqlite3_free(errstr); + return 0; + } + return 1; +} + +int zpm_commit(struct zpm *z) { + char *errstr = 0; + sqlite3_exec(z->db, "commit;", NULL, NULL, &errstr); + if (errstr) { + fprintf(stderr, "sqlite commit error: %s\n", errstr); + sqlite3_free(errstr); + return 0; + } + return 1; +} + +int zpm_rollback(struct zpm *z) { + char *errstr = 0; + sqlite3_exec(z->db, "rollback;", NULL, NULL, &errstr); + if (errstr) { + fprintf(stderr, "sqlite rollback error: %s\n", errstr); + sqlite3_free(errstr); + return 0; + } + return 1; +} + +/* NULL? Create? */ +int zpm_open(struct zpm *pkg, char *path) { + int rc; + char *errstr = 0; + sqlite3 *db = 0; + + pkg->db = 0; + pkg->path = 0; + pkg->version = 0; + pkg->release = 0; + pkg->pkgname = 0; + pkg->installed = 0; + + /* TODO some way to determine if the DB is newly created ? */ + /* could check for tables, if there are any, then check version, etc */ + rc = sqlite3_open(path, &db); + if (rc) { + SQLERROR(sqlite3_errmsg(db)); + sqlite3_close(db); + return 0; + } + sqlite3_exec(pkg->db, "pragma foreign_keys = ON;", NULL, NULL, &errstr); + if (errstr) { + fprintf(stderr, "sqlite foreign key error: %s\n", errstr); + sqlite3_free(errstr); + sqlite3_close(db); + return 0; + } + + pkg->path = dupstr(path); + pkg->db = db; + + /* TODO if this is a new database, create structures */ + + /* get a package. what if more than one? what if none? */ + return 1; +} + +int zpm_close(struct zpm *pkg) { + if (pkg) { + sqlite3_close(pkg->db); + free(pkg->path); + } + /* TODO free any malloced names and such */ + return 1; +} + +int zpm_extract(struct zpm *pkg, char *hash, char *path, int mode) { + int rc; + + int blobsize; + int64_t size; + void *xzdata; + int type; + FILE *out; + sqlite3_stmt *ifile; + + /* TODO check null */ + sqlite3 *db = pkg->db; + + rc = sqlite3_prepare(db, "select size, content from files where hash = ?", -1, &ifile,0); + if (rc != SQLITE_OK) { + SQLERROR(sqlite3_errmsg(db)); + return 0; + } + + /* hash, filename */ + + sqlite3_bind_text(ifile, 1, hash, 64, SQLITE_STATIC); + + rc = sqlite3_step(ifile); + + if (rc == SQLITE_DONE) { + /* didn't find a row */ + sqlite3_finalize(ifile); + sqlite3_close(db); + fprintf(stderr, "no such hash\n"); + return 0; + } + /* either way we're done with this now */ + + if (rc != SQLITE_ROW) { + SQLERROR(sqlite3_errmsg(db)); + sqlite3_finalize(ifile); + sqlite3_close(db); + return 0; + } + + type = sqlite3_column_type(ifile, 0); + if (type == SQLITE_NULL) { + fprintf(stderr, "no file size\n"); + sqlite3_finalize(ifile); + sqlite3_close(db); + return 0; + } + type = sqlite3_column_type(ifile, 1); + if (type == SQLITE_NULL) { + fprintf(stderr, "no file data\n"); + sqlite3_finalize(ifile); + sqlite3_close(db); + return 0; + } + size = sqlite3_column_int64(ifile, 0); + xzdata = (void *)sqlite3_column_blob(ifile, 1); + blobsize = sqlite3_column_bytes(ifile, 1); + + out = fopen(path, "w"); + if (!out) { + fprintf(stderr, "can't open output file %s\n", path); + sqlite3_finalize(ifile); + sqlite3_close(db); + return 5; + } + //fwrite(xzdata, blobsize, 1, stdout); + + fprintf(stderr, "uncompressing %d bytes at %p, expect %lld\n", blobsize, xzdata, (long long int)size); + uncompresslzma(xzdata, blobsize, out); + fclose(out); + + sqlite3_finalize(ifile); + + return 0; + +} + +#if 1 +int zpm_import(struct zpm *pkg, char *path, uint32_t flags, char *hash) { + int fd; + void *content; + struct stat sbuf; + unsigned char tmp[32]; + hash_state md; + sqlite3_stmt *ifile; + int haverow,havedata; + int j,rc,type; + char hashbuf[65]; + + /* xz compress it */ + size_t outlen = 0; + void *outbuf; + + if (!pkg || !pkg->db || !path) { + return 0; + } + + /* use local if caller didn't pass in room */ + if (!hash) { + hash = hashbuf; + } + + /* mmap the file */ + fd = open(path, O_RDONLY); + if (fd == -1) { + return 0; + } + if (fstat(fd, &sbuf) == -1) { + return 0; + } + /* not a regular file? */ + if (!S_ISREG(sbuf.st_mode)) { + /* TODO this is ok, just stored differently */ + return 0; + } + + content = mmap(0, sbuf.st_size, PROT_READ,MAP_PRIVATE, fd, 0); + if (!content) { + return 0; + } + + /* get hash */ + sha256_init(&md); + sha256_process(&md, content, sbuf.st_size); + sha256_done(&md, tmp); + for (j=0;j<32;j++) { + sprintf(hash+j*2, "%02x", (unsigned)tmp[j]); + } + hash[64] = 0; +// fprintf(stderr, "file %s: %s\n", path, hash); + + /* compress */ + outbuf = compresslzma(content, sbuf.st_size, &outlen); +// fprintf(stderr, "compressed to %zu\n", outlen); + + /* don't need the original file now */ + munmap(content, sbuf.st_size); + close(fd); + + /* prepare and bind */ + /* TODO check null */ + sqlite3 *db = pkg->db; + + rc = sqlite3_prepare(db, "select size, content is not null from files where hash = ?", -1, &ifile,0); + if (rc != SQLITE_OK) { + SQLERROR(sqlite3_errmsg(db)); + return 1; + } + + /* hash, filename */ + + sqlite3_bind_text(ifile, 1, hash, 64, SQLITE_STATIC); + + rc = sqlite3_step(ifile); + + if (rc != SQLITE_DONE) { + if (rc != SQLITE_ROW) { + /* didn't find a row */ + SQLERROR(sqlite3_errmsg(db)); + zpm_rollback(pkg); + return 0; + } + haverow = 1; +// fprintf(stderr, "have row for hash\n"); + type = sqlite3_column_type(ifile, 0); + if (type == SQLITE_NULL) { + /* TODO assert, this shouldn't be possible? */ + fprintf(stderr, "no file size\n"); + sqlite3_finalize(ifile); + return 0; + } + type = sqlite3_column_type(ifile, 1); + if (type == SQLITE_NULL) { + /* TODO assert, this shouldn't be possible? */ + fprintf(stderr, "no file data\n"); + sqlite3_finalize(ifile); + return 0; + /* which is fine, just need to update the row then */ + } + havedata = sqlite3_column_int(ifile, 1); + } + + sqlite3_finalize(ifile); + + if (!havedata) { + /* start a transaction */ + // do that outside of here + //zpm_begin(pkg); + + /* insert */ + if (haverow) { + rc = sqlite3_prepare(db, "update files set size = ?, content = ? where hash = ?", -1, &ifile,0); + } else { +// fprintf(stderr, "missing file data\n"); + rc = sqlite3_prepare(db, "insert into files (size, content, hash) values (?,?,?)", -1, &ifile,0); + } + if (rc != SQLITE_OK) { + SQLERROR(sqlite3_errmsg(db)); + fprintf(stderr, "cant prepare data\n"); + zpm_rollback(pkg); + return 0; + } + + sqlite3_bind_int64(ifile, 1, (sqlite3_int64)sbuf.st_size); + if (rc != SQLITE_OK) { + SQLERROR(sqlite3_errmsg(db)); + fprintf(stderr, "cant bind size\n"); + zpm_rollback(pkg); + return 0; + } + sqlite3_bind_blob64(ifile, 2, outbuf, (sqlite3_int64)outlen, SQLITE_STATIC); + if (rc != SQLITE_OK) { + SQLERROR(sqlite3_errmsg(db)); + fprintf(stderr, "cant bind content\n"); + zpm_rollback(pkg); + return 0; + } + sqlite3_bind_text(ifile, 3, hash, 64, SQLITE_STATIC); + if (rc != SQLITE_OK) { + SQLERROR(sqlite3_errmsg(db)); + fprintf(stderr, "cant bind hash\n"); + zpm_rollback(pkg); + return 0; + } + rc = sqlite3_step(ifile); + if (rc != SQLITE_DONE) { + SQLERROR(sqlite3_errmsg(db)); + sqlite3_finalize(ifile); + zpm_rollback(pkg); + return 0; + } + sqlite3_finalize(ifile); + + /* commit */ + //zpm_commit(pkg); + + } + + /* if package and not nopackage flag, add to package */ + if (pkg->pkgname && (!ZPM_NOPACKAGE)) { + /* TODO */ + } + + /* return */ + return 1; +} +#endif + +#if 0 +int main(int ac, char **av){ + sqlite3 *db = 0; + int rc; + + int blobsize; + int64_t size; + void *xzdata; + int type; + FILE *out; + sqlite3_stmt *ifile; + + char *hash; + char *filename; + + if (ac < 3) { + fprintf(stderr, "usage: db hash file\n"); + return 1; + } + + rc = sqlite3_open(av[1], &db); + if (rc) { + SQLERROR(sqlite3_errmsg(db)); + sqlite3_close(db); + return 1; + } + +} +#endif + +#if 0 +Packages are sqlite databases + +get application id and userver + +Primitive operations: + +add path to repo +associate path with package +associate blob with path? +add blob to repo +* extract blob to a path +compare blob to filesystem path +create package with info + +Extra primitives: + +record elf information about blob +compress blob +uncompress blob +sign a package? What are we verifying? +#endif diff --git a/uncompress.c b/uncompress.c new file mode 100644 index 0000000..8b5d98f --- /dev/null +++ b/uncompress.c @@ -0,0 +1,198 @@ +#include +#include +#include +#include +#include +#include + +#include + +#include + +#include "sha256.h" + +#include "lzma.h" + +void uncompresslzma(void *buf, size_t bufsize, FILE *out) { + lzma_stream s = LZMA_STREAM_INIT; + lzma_stream *strm; + + uint8_t outbuf[BUFSIZ]; + + int ret; + + strm = &s; + + ret = lzma_stream_decoder(strm, UINT64_MAX, 0); + /* The only reasonable error here is LZMA_MEM_ERROR. */ + if (ret != LZMA_OK) { + fprintf(stderr, "%s", ret == LZMA_MEM_ERROR ? strerror(ENOMEM) + : "Internal error (bug)"); + exit(EXIT_FAILURE); + } + + strm->avail_in = bufsize; + strm->next_in = buf; + strm->avail_out = BUFSIZ; + strm->next_out = outbuf; + + lzma_action action = LZMA_RUN; + + while (1) { + ret = lzma_code(strm, action); + + // Write and check write error before checking decoder error. + // This way as much data as possible gets written to output + // even if decoder detected an error. + if (strm->avail_out == 0 || ret != LZMA_OK) { + const size_t write_size = BUFSIZ - strm->avail_out; + + if (fwrite(outbuf, 1, write_size, out) != write_size) { + // Wouldn't be a surprise if writing to stderr + // would fail too but at least try to show an + // error message. + fprintf(stderr, "Cannot write to output file stream: " + "%s", strerror(errno)); + exit(EXIT_FAILURE); + } + + strm->next_out = outbuf; + strm->avail_out = BUFSIZ; + } + + if (ret != LZMA_OK) { + if (ret == LZMA_STREAM_END) { + // lzma_stream_decoder() already guarantees + // that there's no trailing garbage. + assert(strm->avail_in == 0); + //assert(action == LZMA_FINISH); + return; + } + + const char *msg; + switch (ret) { + case LZMA_MEM_ERROR: + msg = strerror(ENOMEM); + break; + + case LZMA_FORMAT_ERROR: + msg = "File format not recognized"; + break; + + case LZMA_OPTIONS_ERROR: + // FIXME: Better message? + msg = "Unsupported compression options"; + break; + + case LZMA_DATA_ERROR: + msg = "File is corrupt"; + break; + + case LZMA_BUF_ERROR: + msg = "Unexpected end of input"; + break; + + default: + msg = "Internal error (bug)"; + break; + } + + fprintf(stderr, "xz: %s\n", msg); + exit(EXIT_FAILURE); + } + } +} + +#define SQLERROR(x) fprintf(stderr, "%s %d: %s\n", __func__, __LINE__, (x)) + +int main(int ac, char **av){ + sqlite3 *db = 0; + int rc; + + int blobsize; + int64_t size; + void *xzdata; + int type; + FILE *out; + sqlite3_stmt *ifile; + + char *hash; + char *filename; + + if (ac < 3) { + fprintf(stderr, "usage: db hash file\n"); + return 1; + } + + rc = sqlite3_open(av[1], &db); + if (rc) { + SQLERROR(sqlite3_errmsg(db)); + sqlite3_close(db); + return 1; + } + + rc = sqlite3_prepare(db, "select size, content from files where hash = ?", -1, &ifile,0); + if (rc != SQLITE_OK) { + SQLERROR(sqlite3_errmsg(db)); + return 1; + } + + /* hash, filename */ + hash = av[2]; + filename = av[3]; + + sqlite3_bind_text(ifile, 1, hash, 64, SQLITE_STATIC); + + rc = sqlite3_step(ifile); + + if (rc == SQLITE_DONE) { + /* didn't find a row */ + sqlite3_finalize(ifile); + sqlite3_close(db); + fprintf(stderr, "no such hash\n"); + return 1; + } + /* either way we're done with this now */ + + if (rc != SQLITE_ROW) { + SQLERROR(sqlite3_errmsg(db)); + sqlite3_finalize(ifile); + sqlite3_close(db); + return 2; + } + + type = sqlite3_column_type(ifile, 0); + if (type == SQLITE_NULL) { + fprintf(stderr, "no file size\n"); + sqlite3_finalize(ifile); + sqlite3_close(db); + return 3; + } + type = sqlite3_column_type(ifile, 1); + if (type == SQLITE_NULL) { + fprintf(stderr, "no file data\n"); + sqlite3_finalize(ifile); + sqlite3_close(db); + return 4; + } + size = sqlite3_column_int64(ifile, 0); + xzdata = (void *)sqlite3_column_blob(ifile, 1); + blobsize = sqlite3_column_bytes(ifile, 1); + + out = fopen(filename, "w"); + if (!out) { + fprintf(stderr, "can't open output file %s\n", filename); + sqlite3_finalize(ifile); + sqlite3_close(db); + return 5; + } + //fwrite(xzdata, blobsize, 1, stdout); + + fprintf(stderr, "uncompressing %d bytes at %p, expect %lld\n", blobsize, xzdata, (long long int)size); + uncompresslzma(xzdata, blobsize, out); + fclose(out); + + sqlite3_finalize(ifile); + sqlite3_close(db); + return 0; +} diff --git a/zpm-addfile.c b/zpm-addfile.c new file mode 100644 index 0000000..961b6b8 --- /dev/null +++ b/zpm-addfile.c @@ -0,0 +1,20 @@ +#include +#include "zpm.h" + +int main(int ac, char **av){ + struct zpm pkg; + char hash[65]; + + if (ac < 3) { + fprintf(stderr, "usage: db path\n"); + return 1; + } + zpm_open(&pkg, av[1]); + zpm_begin(&pkg); + zpm_import(&pkg, av[2], 0, hash); + zpm_commit(&pkg); + zpm_close(&pkg); + //fprintf(stdout, "%s %s\n", hash, av[2]); + fprintf(stdout, "%s\n", hash); + return 0; +} diff --git a/zpm-addtopackage b/zpm-addtopackage new file mode 100755 index 0000000..3833c00 --- /dev/null +++ b/zpm-addtopackage @@ -0,0 +1,96 @@ +#!/bin/sh + +package=${1:-$ZPMPACKAGE} +shift +pkgver=${ZPMPACKAGEVER:-1.0} +pkgrel=${ZPMPACKAGEREL:-1} + +# option for "multipackage" just to let the system know that's what you meant +# option to take filenames from stdin +# parse package, version, release from file if not given +while getopts :f:v:r:d:a:u:l:p:b:P: opt; do + case $opt in + f) pkgfile="$OPTARG" ;; + v) pkgver="$OPTARG" ;; + r) pkgrel="$OPTARG" ;; + d) description="$OPTARG" ;; + a) arch="$OPTARG" ;; + u) url="$OPTARG" ;; + l) licenses="$OPTARG" ;; + p) packager="$OPTARG" ;; + b) builddate="$OPTARG" ;; + P) prefix="$OPTARG" ;; + esac +done + +set -e +if [ -z "$pkgfile" ]; then + pkgfile="$package-$pkgver-$pkgrel.zpm" +fi + +if [ ! -f $pkgfile ]; then + ./zpm-newpackage $package || exit 1 +else + appid=$(sqlite3 $pkgfile 'pragma application_id;' | ( echo obase = 16; cat - ) | bc) + if [ "$appid" != "5A504442" ]; then + echo $pkgfile does not appear to be a zpm package file + exit 1 + fi +fi + +for path in $*; do + mtime=$(stat -c '%Y' $path) + uid=$(stat -c '%u' $path) + gid=$(stat -c '%g' $path) + username=$(stat -c '%U' $path) + groupname=$(stat -c '%G' $path) + mode=$(stat -c '%a' $path) + + # strip off leading slashes + rpath=$(echo "$path" | sed -e 's|^/*||') + # and a leading ./ + rpath=${rpath#./} + rpath=$(echo "$rpath" | sed -e 's|^/*||') + + if [ -z "$rpath" ] || [ "$rpath" = '.' ]; then + continue + fi + + if [ ! -z "$prefix" ]; then + # trailing slashes on prefix + prefix=$(echo "$prefix" | sed -e 's|/*$||') + rpath="$prefix/$rpath" + fi + + if [ -f "$path" ]; then + + hash=$(./zpm-addfile $pkgfile $path) + +#if [ -z "$hash" ]; then continue; fi + +# TODO mtime, mode +sqlite3 $pkgfile < $target"} +printf "%s\n" $path +done diff --git a/zpm-extract.c b/zpm-extract.c new file mode 100644 index 0000000..e050bf2 --- /dev/null +++ b/zpm-extract.c @@ -0,0 +1,118 @@ +#include +#include +#include +#include +#include +#include + +#include + +#include "zpm.h" + +#if 1 +int main(int ac, char **av){ + struct zpm pkg; + int mode = 0644; + + if (ac < 3) { + fprintf(stderr, "usage: db hash file\n"); + return 1; + } + zpm_open(&pkg, av[1]); + zpm_extract(&pkg, av[2], av[3], mode); + zpm_close(&pkg); +} +#else + +int main(int ac, char **av) { + sqlite3 *db = 0; + int rc; + + int blobsize; + int64_t size; + void *xzdata; + int type; + FILE *out; + sqlite3_stmt *ifile; + + char *hash; + char *filename; + + if (ac < 3) { + fprintf(stderr, "usage: db hash file\n"); + return 1; + } + + rc = sqlite3_open(av[1], &db); + if (rc) { + SQLERROR(sqlite3_errmsg(db)); + sqlite3_close(db); + return 1; + } + + rc = sqlite3_prepare(db, "select size, content from files where hash = ?", -1, &ifile,0); + if (rc != SQLITE_OK) { + SQLERROR(sqlite3_errmsg(db)); + return 1; + } + + /* hash, filename */ + hash = av[2]; + filename = av[3]; + + sqlite3_bind_text(ifile, 1, hash, 64, SQLITE_STATIC); + + rc = sqlite3_step(ifile); + + if (rc == SQLITE_DONE) { + /* didn't find a row */ + sqlite3_finalize(ifile); + sqlite3_close(db); + fprintf(stderr, "no such hash\n"); + return 1; + } + /* either way we're done with this now */ + + if (rc != SQLITE_ROW) { + SQLERROR(sqlite3_errmsg(db)); + sqlite3_finalize(ifile); + sqlite3_close(db); + return 2; + } + + type = sqlite3_column_type(ifile, 0); + if (type == SQLITE_NULL) { + fprintf(stderr, "no file size\n"); + sqlite3_finalize(ifile); + sqlite3_close(db); + return 3; + } + type = sqlite3_column_type(ifile, 1); + if (type == SQLITE_NULL) { + fprintf(stderr, "no file data\n"); + sqlite3_finalize(ifile); + sqlite3_close(db); + return 4; + } + size = sqlite3_column_int64(ifile, 0); + xzdata = (void *)sqlite3_column_blob(ifile, 1); + blobsize = sqlite3_column_bytes(ifile, 1); + + out = fopen(filename, "w"); + if (!out) { + fprintf(stderr, "can't open output file %s\n", filename); + sqlite3_finalize(ifile); + sqlite3_close(db); + return 5; + } + //fwrite(xzdata, blobsize, 1, stdout); + + fprintf(stderr, "uncompressing %d bytes at %p, expect %lld\n", blobsize, xzdata, (long long int)size); + uncompresslzma(xzdata, blobsize, out); + fclose(out); + + sqlite3_finalize(ifile); + sqlite3_close(db); + return 0; +} +#endif diff --git a/zpm-extractfile.c b/zpm-extractfile.c new file mode 100644 index 0000000..68ab0e1 --- /dev/null +++ b/zpm-extractfile.c @@ -0,0 +1,381 @@ +/* + * add a file/files to an sqlite db + * in the 'files' table. + */ + +#include +#include +#include +#include +#include +#include + +#include + +#include +#include "sha256.h" + +#include "lzma.h" + + +void uncompresslzma(void *buf, size_t bufsize, FILE *out) { + lzma_stream s = LZMA_STREAM_INIT; + lzma_stream *strm; + + uint8_t outbuf[BUFSIZ]; + + int ret; + + strm = &s; + + ret = lzma_stream_decoder(strm, UINT64_MAX, 0); + /* The only reasonable error here is LZMA_MEM_ERROR. */ + if (ret != LZMA_OK) { + fprintf(stderr, "%s", ret == LZMA_MEM_ERROR ? strerror(ENOMEM) + : "Internal error (bug)"); + exit(EXIT_FAILURE); + } + + strm->avail_in = bufsize; + strm->next_in = buf; + strm->avail_out = BUFSIZ; + strm->next_out = outbuf; + + lzma_action action = LZMA_RUN; + + while (1) { + ret = lzma_code(strm, action); + + // Write and check write error before checking decoder error. + // This way as much data as possible gets written to output + // even if decoder detected an error. + if (strm->avail_out == 0 || ret != LZMA_OK) { + const size_t write_size = BUFSIZ - strm->avail_out; + + if (fwrite(outbuf, 1, write_size, out) != write_size) { + // Wouldn't be a surprise if writing to stderr + // would fail too but at least try to show an + // error message. + fprintf(stderr, "Cannot write to output file stream: " + "%s", strerror(errno)); + exit(EXIT_FAILURE); + } + + strm->next_out = outbuf; + strm->avail_out = BUFSIZ; + } + + if (ret != LZMA_OK) { + if (ret == LZMA_STREAM_END) { + // lzma_stream_decoder() already guarantees + // that there's no trailing garbage. + assert(strm->avail_in == 0); + //assert(action == LZMA_FINISH); + return; + } + + const char *msg; + switch (ret) { + case LZMA_MEM_ERROR: + msg = strerror(ENOMEM); + break; + + case LZMA_FORMAT_ERROR: + msg = "File format not recognized"; + break; + + case LZMA_OPTIONS_ERROR: + // FIXME: Better message? + msg = "Unsupported compression options"; + break; + + case LZMA_DATA_ERROR: + msg = "File is corrupt"; + break; + + case LZMA_BUF_ERROR: + msg = "Unexpected end of input"; + break; + + default: + msg = "Internal error (bug)"; + break; + } + + fprintf(stderr, "xz: %s\n", msg); + exit(EXIT_FAILURE); + } + } +} + +#define SQLERROR(x) fprintf(stderr, "%s %d: %s\n", __func__, __LINE__, (x)) + +int main(int ac, char **av){ + sqlite3 *db = 0; + int rc; + + int blobsize; + int64_t size; + void *xzdata; + int type; + FILE *out; + sqlite3_stmt *ifile; + + char *hash; + char *filename; + + if (ac < 3) { + fprintf(stderr, "usage: db hash file\n"); + return 1; + } + + rc = sqlite3_open(av[1], &db); + if (rc) { + SQLERROR(sqlite3_errmsg(db)); + sqlite3_close(db); + return 1; + } + + rc = sqlite3_prepare(db, "select size, content from files where hash = ?", -1, &ifile,0); + if (rc != SQLITE_OK) { + SQLERROR(sqlite3_errmsg(db)); + return 1; + } + + /* hash, filename */ + hash = av[2]; + filename = av[3]; + + sqlite3_bind_text(ifile, 1, hash, 64, SQLITE_STATIC); + + rc = sqlite3_step(ifile); + + if (rc == SQLITE_DONE) { + /* didn't find a row */ + sqlite3_finalize(ifile); + sqlite3_close(db); + fprintf(stderr, "no such hash\n"); + return 1; + } + /* either way we're done with this now */ + + if (rc != SQLITE_ROW) { + SQLERROR(sqlite3_errmsg(db)); + sqlite3_finalize(ifile); + sqlite3_close(db); + return 2; + } + + type = sqlite3_column_type(ifile, 0); + if (type == SQLITE_NULL) { + fprintf(stderr, "no file size\n"); + sqlite3_finalize(ifile); + sqlite3_close(db); + return 3; + } + type = sqlite3_column_type(ifile, 1); + if (type == SQLITE_NULL) { + fprintf(stderr, "no file data\n"); + sqlite3_finalize(ifile); + sqlite3_close(db); + return 4; + } + size = sqlite3_column_int64(ifile, 0); + xzdata = (void *)sqlite3_column_blob(ifile, 1); + blobsize = sqlite3_column_bytes(ifile, 1); + + out = fopen(filename, "w"); + if (!out) { + fprintf(stderr, "can't open output file %s\n", filename); + sqlite3_finalize(ifile); + sqlite3_close(db); + return 5; + } + //fwrite(xzdata, blobsize, 1, stdout); + + fprintf(stderr, "uncompressing %d bytes at %p, expect %lld\n", blobsize, xzdata, (long long int)size); + uncompresslzma(xzdata, blobsize, out); + fclose(out); + + sqlite3_finalize(ifile); + sqlite3_close(db); + return 0; +} +void *compresslzma(void *buf, size_t bufsize, size_t *len) { + /* xz compress it */ + size_t outsize; + void *outbuf; + size_t outlen = 0; + + outsize = lzma_stream_buffer_bound(bufsize); + outbuf = malloc(outsize); + if (!outbuf) { + *len = 0; + return NULL; + } + /* TODO adjust encoding level for size */ + lzma_easy_buffer_encode(6, LZMA_CHECK_CRC64, NULL, + buf, bufsize, + outbuf, &outlen, outsize + ); + *len = outlen; + return outbuf; +} + +static int callback(void *NotUsed, int argc, char **argv, char **azColName){ + int i; + for(i=0; i\n' + exit 1 + ;; + esac +done + +pkgname="$1" + +pacman -Qlq "$pkgname" | while read file; do + if [ ! -f "$file" ]; then continue; fi + base=$(basename "$file") + if [ $all -eq 0 ] && [ "$base" = "${base#lib}" ]; then continue; fi + + if [ $check -eq 1 ]; then + elftype -e "$file" + rv=$? + if [ $rv -ne 0 ]; then continue; fi + for lib in $(zpm getlibs $file | grep preserve); do + echo $pkgname $file $lib + done + continue + fi + + soname=$(soname $file) + if [ $? -eq 0 ] && [ -n "$soname" ]; then + dir=$(dirname "$file") + if [ "$script" -ne 0 ]; then + printf 'mkdir -p "%s"\n' "$dir/preserve" + printf 'ln -f "%s" "%s"\n' "$file" "$dir/preserve/" + else + mkdir -p "$dir/preserve" + if [ "$verbose" -gt 0 ]; then + printf '# preserving %s\n' "$file" + fi + ln -f "$file" "$dir/preserve/" + fi + fi +done diff --git a/zpm-install b/zpm-install new file mode 100755 index 0000000..e722bf3 --- /dev/null +++ b/zpm-install @@ -0,0 +1,102 @@ +#!/bin/sh + +package=${1:-$ZPMPACKAGE} +shift +pkgver=${ZPMPACKAGEVER:-1.0} +pkgrel=${ZPMPACKAGEREL:-1} + +pkgroot=/ + +# option for "multipackage" just to let the system know that's what you meant +# option to take filenames from stdin +# parse package, version, release from file if not given +while getopts :f:v:r:d:a:u:l:p:b:P: opt; do + case $opt in + R) pkgroot="$OPTARG" ;; + f) pkgfile="$OPTARG" ;; + v) pkgver="$OPTARG" ;; + r) pkgrel="$OPTARG" ;; + d) description="$OPTARG" ;; + a) arch="$OPTARG" ;; + u) url="$OPTARG" ;; + l) licenses="$OPTARG" ;; + p) packager="$OPTARG" ;; + b) builddate="$OPTARG" ;; + P) prefix="$OPTARG" ;; + esac +done + +set -e +if [ -z "$pkgfile" ]; then + pkgfile="$package-$pkgver-$pkgrel.zpm" +fi + +appid=$(sqlite3 $pkgfile 'pragma application_id;' | ( echo obase = 16; cat - ) | bc) +if [ "$appid" != "5A504442" ]; then + echo $pkgfile does not appear to be a zpm package file + exit 1 +fi + +# check if package exists +# run preinstall or preupgrade stage, in chroot + +# add package info and mark installing to local package database +# each path: add to local db, extract, set mode/owner/mtime etc. +# mark install done in local database + +for path in $*; do + mtime=$(stat -c '%Y' $path) + uid=$(stat -c '%u' $path) + gid=$(stat -c '%g' $path) + username=$(stat -c '%U' $path) + groupname=$(stat -c '%G' $path) + mode=$(stat -c '%a' $path) + + # strip off leading slashes + rpath=$(echo "$path" | sed -e 's|^/*||') + # and a leading ./ + rpath=${rpath#./} + rpath=$(echo "$rpath" | sed -e 's|^/*||') + + if [ -z "$rpath" ] || [ "$rpath" = '.' ]; then + continue + fi + + if [ ! -z "$prefix" ]; then + # trailing slashes on prefix + prefix=$(echo "$prefix" | sed -e 's|/*$||') + rpath="$prefix/$rpath" + fi + + if [ -f "$path" ]; then + + hash=$(./zpm-addfile $pkgfile $path) + +#if [ -z "$hash" ]; then continue; fi + +# TODO mtime, mode +sqlite3 $pkgfile < $target"} +printf "%s\n" $path +done diff --git a/zpm-ipkgfile b/zpm-ipkgfile new file mode 100755 index 0000000..0f9440c --- /dev/null +++ b/zpm-ipkgfile @@ -0,0 +1,10 @@ +#!/bin/sh + +pkgfile=$1 +shift + +for path in $*; do + mkdir -p tmp/$(dirname $path) + fhash=$(sqlite3 $pkgfile "select hash from packagefiles where path = '$path';") + ./zpm-extract $pkgfile $fhash ./tmp/$path +done diff --git a/zpm-newpackage b/zpm-newpackage new file mode 100755 index 0000000..fbd44b5 --- /dev/null +++ b/zpm-newpackage @@ -0,0 +1,37 @@ +#!/bin/sh + +package=${1:-$ZPMPACKAGE} +pkgver=${ZPMPACKAGEVER:-1.0} +pkgrel=${ZPMPACKAGEREL:-1} + +builddate=$(date '+%s') + +while getopts :f:v:r:d:a:u:l:p:b: opt; do + case $opt in + f) pkgfile="$OPTARG" ;; + v) pkgver="$OPTARG" ;; + r) pkgrel="$OPTARG" ;; + d) description="$OPTARG" ;; + a) arch="$OPTARG" ;; + u) url="$OPTARG" ;; + l) licenses="$OPTARG" ;; + p) packager="$OPTARG" ;; + b) builddate="$OPTARG" ;; + esac +done + +set -e + +if [ -z "$pkgfile" ]; then + pkgfile="$package-$pkgver-$pkgrel.zpm" +fi + +if [ ! -e $pkgfile ]; then + sqlite3 $pkgfile < db.sql +fi + +sqlite3 $pkgfile <&2 + exit 1 +fi + +for path in $*; do +hash=$(./zpm-addfile $pkgfile $path) + +sqlite3 $pkgfile <\n' + exit 1 + ;; + esac +done pkgname="$1" pacman -Qlq "$pkgname" | while read file; do if [ ! -f "$file" ]; then continue; fi + base=$(basename "$file") + if [ $all -eq 0 ] && [ "$base" = "${base#lib}" ]; then continue; fi + + if [ $check -eq 1 ]; then + elftype -e "$file" + rv=$? + if [ $rv -ne 0 ]; then continue; fi + for lib in $(zpm getlibs $file | grep preserve); do + echo $pkgname $file $lib + done + continue + fi soname=$(soname $file) if [ $? -eq 0 ] && [ -n "$soname" ]; then dir=$(dirname "$file") - mkdir -p "$dir/preserve" - if [ "$verbose" -gt 0 ]; then - printf 'preserving %s\n' "$file" + if [ "$script" -ne 0 ]; then + printf 'mkdir -p "%s"\n' "$dir/preserve" + printf 'ln -f "%s" "%s"\n' "$file" "$dir/preserve/" + else + mkdir -p "$dir/preserve" + if [ "$verbose" -gt 0 ]; then + printf '# preserving %s\n' "$file" + fi + ln -f "$file" "$dir/preserve/" fi - ln -f "$file" "$dir/preserve/" fi done diff --git a/zpm-showpkg b/zpm-showpkg new file mode 100755 index 0000000..2322f6f --- /dev/null +++ b/zpm-showpkg @@ -0,0 +1,13 @@ +#!/bin/sh + +{ +for pkgfile in $*; do + + sqlite3 $pkgfile < +#include +#include + +struct zpm { + sqlite3 *db; + char *path; /* path to package file */ + char *version; + int release; + char *pkgname; + time_t installed; /* install time, 0 for not installed */ +}; + +struct zpm_file { + char *path; + int mode; + char *tags; + char *owner; + char *group; + time_t mtime; + struct zpm_file *next; /* so you can make a linked list */ +}; + +/* NULL? Create? */ +int zpm_open(struct zpm *pkg, char *path); +int zpm_pkgname(char *base, char *version, int release); /* construct a package file name */ + +/* flags for preserving mode, owner, etc */ +/* puts hash of import in hash */ +/* path can be a hash, with an "INTERNAL" flag, i.e. internally import */ +#define ZPM_MODE 0x1 +#define ZPM_OWNER 0x2 +#define ZPM_MTIME 0x4 +#define ZPM_INTERNAL 0x8 +#define ZPM_NOBLOB 0x10 +/* don't run scripts on install */ +#define ZPM_NOSCRIPTS 0x10 +/* don't import file to a particular package */ +#define ZPM_NOPACKAGE 0x20 + +int zpm_import(struct zpm *zp, char *path, uint32_t flags, char *hash); + +/* tag a file. relative to "current package" */ +int zpm_tag(struct zpm *zp, char *path, char *tags); +/* should this be broken up into separage functions ? */ +int zpm_md(struct zpm *zp, char *path, int mode, char *owner, char *group, time_t mtime); + +/* export hash to dest */ +int zpm_extract(struct zpm *pkg, char *hash, char *path, int mode); + +/* export path to dest */ +int zpm_export(struct zpm *zp, char *path, uint32_t flags, char *dest); + +int zpm_close(struct zpm *zp); + +/* attach a signature to a package */ +int zpm_sign(struct zpm *z, size_t s, void *signature); + +/* set the package info to the nth package, -1 to return count? */ +/* further import/exports and such will be relative to this package */ +int zpm_package(struct zpm *zp, int n); + +/* get file information */ +int zpm_stat(struct zpm *z, struct zpm_file *f, int n); + +/* will also set the package context to the new package */ +int zpm_newpkg(struct zpm *z, char *base, char *version, int release); + +/* transactions */ +int zpm_begin(struct zpm *z); +int zpm_commit(struct zpm *z); +int zpm_rollback(struct zpm *z); + +/* higher level operations */ + +/* install or uninstall the package */ +/* flag for keeping the blobs in local */ +/* what about tag filtering */ +int zpm_install(struct zpm *z, struct zpm *local, uint32_t flags); +int zpm_uninstall(struct zpm *local); + +/* slurp up the actual blobs */ +/* what about versioning them if they change */ +int zpm_preserve(struct zpm *local); + +/* check file integrity */ +int zpm_checkinstall(struct zpm *local); + +int zpm_merge(struct zpm *z, struct zpm *src, uint32_t flags); + +void uncompresslzma(void *buf, size_t bufsize, FILE *out); +void *compresslzma(void *buf, size_t bufsize, size_t *len); +#define SQLERROR(x) fprintf(stderr, "%s %d: %s\n", __func__, __LINE__, (x)) -- 2.40.0