/* This file is part of "reprepro" * Copyright (C) 2006,2007,2008,2009 Bernhard R. Link * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License version 2 as * published by the Free Software Foundation. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License * along with this program; if not, write to the Free Software * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02111-1301 USA */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "error.h" #include "ignore.h" #include "mprintf.h" #include "filecntl.h" #include "strlist.h" #include "dirs.h" #include "names.h" #include "checksums.h" #include "chunks.h" #include "target.h" #include "signature.h" #include "binaries.h" #include "sources.h" #include "dpkgversions.h" #include "uploaderslist.h" #include "guesscomponent.h" #include "log.h" #include "override.h" #include "tracking.h" #include "incoming.h" #include "files.h" #include "configparser.h" #include "changes.h" enum permitflags { /* do not error out on unused files */ pmf_unused_files = 0, /* do not error out if there already is a newer package */ pmf_oldpackagenewer, pmf_COUNT /* must be last */ }; enum cleanupflags { /* delete everything referenced by a .changes file * when it is not accepted */ cuf_on_deny = 0, /* check owner when deleting on_deny */ cuf_on_deny_check_owner, /* delete everything referenced by a .changes on errors * after accepting that .changes file*/ cuf_on_error, /* delete unused files after sucessfully * processing the used ones */ cuf_unused_files, cuf_COUNT /* must be last */ }; enum optionsflags { /* only put _all.deb comes with those of some architecture, * only put in those architectures */ iof_limit_arch_all = 0, /* allow .changes file to specify multipe distributions */ iof_multiple_distributions, iof_COUNT /* must be last */ }; struct incoming { /* by incoming_parse: */ char *name; char *directory; char *morguedir; char *tempdir; char *logdir; struct strlist allow; struct distribution **allow_into; struct distribution *default_into; /* by incoming_prepare: */ struct strlist files; bool *processed; bool *delete; bool permit[pmf_COUNT]; bool cleanup[cuf_COUNT]; bool options[iof_COUNT]; /* only to ease parsing: */ size_t lineno; }; #define BASENAME(i,ofs) (i)->files.values[ofs] /* the changes file is always the first one listed */ #define changesfile(c) (c->files) static void incoming_free(/*@only@*/ struct incoming *i) { if( i == NULL ) return; free(i->name); free(i->morguedir); free(i->tempdir); free(i->logdir); free(i->directory); strlist_done(&i->allow); free(i->allow_into); strlist_done(&i->files); free(i->processed); free(i->delete); free(i); } static retvalue incoming_prepare(struct incoming *i) { DIR *dir; struct dirent *ent; retvalue r; int ret; /* TODO: decide whether to clean this directory first ... */ r = dirs_make_recursive(i->tempdir); if( RET_WAS_ERROR(r) ) return r; dir = opendir(i->directory); if( dir == NULL ) { int e = errno; fprintf(stderr, "Cannot scan '%s': %s\n", i->directory, strerror(e)); return RET_ERRNO(e); } while( (ent = readdir(dir)) != NULL ) { if( ent->d_name[0] == '.' ) continue; /* this should be impossible to hit. * but given utf-8 encoding filesystems and * overlong slashes, better check than be sorry */ if( strchr(ent->d_name, '/') != NULL ) continue; r = strlist_add_dup(&i->files, ent->d_name) ; if( RET_WAS_ERROR(r) ) { (void)closedir(dir); return r; } } ret = closedir(dir); if( ret != 0 ) { int e = errno; fprintf(stderr, "Error scanning '%s': %s\n", i->directory, strerror(e)); return RET_ERRNO(e); } i->processed = calloc(i->files.count, sizeof(bool)); if( i->processed == NULL ) return RET_ERROR_OOM; i->delete = calloc(i->files.count, sizeof(bool)); if( i->delete == NULL ) return RET_ERROR_OOM; return RET_OK; } struct read_incoming_data { /*@temp@*/const char *name; /*@temp@*/struct distribution *distributions; struct incoming *i; }; static retvalue translate(struct distribution *distributions, struct strlist *names, struct distribution ***r) { struct distribution **d; int j; d = calloc(names->count,sizeof(struct distribution*)); if( d == NULL ) return RET_ERROR_OOM; for( j = 0 ; j < names->count ; j++ ) { d[j] = distribution_find(distributions, names->values[j]); if( d[j] == NULL ) { free(d); return RET_ERROR; } } *r = d; return RET_OK; } CFstartparse(incoming) { CFstartparseVAR(incoming,result_p); struct incoming *i; i = calloc(1,sizeof(struct incoming)); if( i == NULL ) return RET_ERROR_OOM; *result_p = i; return RET_OK; } CFfinishparse(incoming) { CFfinishparseVARS(incoming,i,last,d); if( !complete || strcmp(i->name, d->name) != 0) { incoming_free(i); return RET_NOTHING; } if( d->i != NULL ) { fprintf(stderr, "Multiple definitions of '%s' within '%s': first started at line %u, second at line %u!\n", d->name, config_filename(iter), (unsigned int)d->i->lineno, config_firstline(iter)); incoming_free(i); incoming_free(d->i); d->i = NULL; return RET_ERROR; } if( i->morguedir != NULL && i->morguedir[0] != '/' ) { char *n = calc_dirconcat(global.basedir, i->morguedir); if( n == NULL ) { incoming_free(i); return RET_ERROR_OOM; } free(i->morguedir); i->morguedir = n; } if( i->tempdir[0] != '/' ) { char *n = calc_dirconcat(global.basedir, i->tempdir); if( n == NULL ) { incoming_free(i); return RET_ERROR_OOM; } free(i->tempdir); i->tempdir = n; } if( i->directory[0] != '/' ) { char *n = calc_dirconcat(global.basedir, i->directory); if( n == NULL ) { incoming_free(i); return RET_ERROR_OOM; } free(i->directory); i->directory = n; } if( i->default_into == NULL && i->allow.count == 0 ) { fprintf(stderr, "There is neither an 'Allow' nor a 'Default' definition in rule '%s'\n" "(starting at line %u, ending at line %u of %s)!\n" "Aborting as nothing would be let in.\n", d->name, config_firstline(iter), config_line(iter), config_filename(iter)); incoming_free(i); return RET_ERROR; } if( i->morguedir != NULL && !i->cleanup[cuf_on_deny] && !i->cleanup[cuf_on_error] && !i->cleanup[cuf_unused_files] ) { fprintf(stderr, "Warning: There is a 'MorgueDir' but no 'Cleanup' to act on in rule '%s'\n" "(starting at line %u, ending at line %u of %s)!\n", d->name, config_firstline(iter), config_line(iter), config_filename(iter)); } d->i = i; i->lineno = config_firstline(iter); /* only suppreses the last unused warning: */ *last = i; return RET_OK; } CFSETPROC(incoming,default) { CFSETPROCVARS(incoming,i,d); char *default_into; retvalue r; r = config_getonlyword(iter, headername, NULL, &default_into); assert( r != RET_NOTHING ); if( RET_WAS_ERROR(r) ) return r; i->default_into = distribution_find(d->distributions, default_into); free(default_into); return ( i->default_into == NULL )?RET_ERROR:RET_OK; } CFSETPROC(incoming,allow) { CFSETPROCVARS(incoming,i,d); struct strlist allow_into; retvalue r; r = config_getsplitwords(iter, headername, &i->allow, &allow_into); assert( r != RET_NOTHING ); if( RET_WAS_ERROR(r) ) return r; assert( i->allow.count == allow_into.count ); r = translate(d->distributions, &allow_into, &i->allow_into); strlist_done(&allow_into); if( RET_WAS_ERROR(r) ) return r; return RET_OK; } CFSETPROC(incoming,permit) { CFSETPROCVARS(incoming,i,d); static const struct constant const permitconstants[] = { { "unused_files", pmf_unused_files}, { "older_version", pmf_oldpackagenewer}, /* not yet implemented: { "downgrade", pmf_downgrade}, */ { NULL, -1} }; if( IGNORABLE(unknownfield) ) return config_getflags(iter, headername, permitconstants, i->permit, true, ""); else if( i->name == NULL ) return config_getflags(iter, headername, permitconstants, i->permit, false, "\n(try put Name: before Permit: to ignore if it is from the wrong rule"); else if( strcmp(i->name, d->name) != 0 ) return config_getflags(iter, headername, permitconstants, i->permit, true, " (but not within the rule we are intrested in.)"); else return config_getflags(iter, headername, permitconstants, i->permit, false, " (use --ignore=unknownfield to ignore this)\n"); } CFSETPROC(incoming,cleanup) { CFSETPROCVARS(incoming,i,d); static const struct constant const cleanupconstants[] = { { "unused_files", cuf_unused_files}, { "on_deny", cuf_on_deny}, /* not yet implemented { "on_deny_check_owner", cuf_on_deny_check_owner}, */ { "on_error", cuf_on_error}, { NULL, -1} }; if( IGNORABLE(unknownfield) ) return config_getflags(iter, headername, cleanupconstants, i->cleanup, true, ""); else if( i->name == NULL ) return config_getflags(iter, headername, cleanupconstants, i->cleanup, false, "\n(try put Name: before Cleanup: to ignore if it is from the wrong rule"); else if( strcmp(i->name, d->name) != 0 ) return config_getflags(iter, headername, cleanupconstants, i->cleanup, true, " (but not within the rule we are intrested in.)"); else return config_getflags(iter, headername, cleanupconstants, i->cleanup, false, " (use --ignore=unknownfield to ignore this)\n"); } CFSETPROC(incoming,options) { CFSETPROCVARS(incoming,i,d); static const struct constant const optionsconstants[] = { { "limit_arch_all", iof_limit_arch_all}, { "multiple_distributions", iof_multiple_distributions}, { NULL, -1} }; if( IGNORABLE(unknownfield) ) return config_getflags(iter, headername, optionsconstants, i->options, true, ""); else if( i->name == NULL ) return config_getflags(iter, headername, optionsconstants, i->options, false, "\n(try put Name: before Options: to ignore if it is from the wrong rule"); else if( strcmp(i->name, d->name) != 0 ) return config_getflags(iter, headername, optionsconstants, i->options, true, " (but not within the rule we are intrested in.)"); else return config_getflags(iter, headername, optionsconstants, i->options, false, " (use --ignore=unknownfield to ignore this)\n"); } CFvalueSETPROC(incoming, name) CFvalueSETPROC(incoming, logdir) CFdirSETPROC(incoming, tempdir) CFdirSETPROC(incoming, morguedir) CFdirSETPROC(incoming, directory) CFtruthSETPROC2(incoming, multiple, options[iof_multiple_distributions]) static const struct configfield incomingconfigfields[] = { CFr("Name", incoming, name), CFr("TempDir", incoming, tempdir), CFr("IncomingDir", incoming, directory), CF("MorgueDir", incoming, morguedir), CF("Default", incoming, default), CF("Allow", incoming, allow), CF("Multiple", incoming, multiple), CF("Options", incoming, options), CF("Cleanup", incoming, cleanup), CF("Permit", incoming, permit), CF("Logdir", incoming, logdir) }; static retvalue incoming_init(struct distribution *distributions, const char *name, /*@out@*/struct incoming **result) { retvalue r; struct read_incoming_data imports; imports.name = name; imports.distributions = distributions; imports.i = NULL; r = configfile_parse("incoming", IGNORABLE(unknownfield), startparseincoming, finishparseincoming, incomingconfigfields, ARRAYCOUNT(incomingconfigfields), &imports); if( RET_WAS_ERROR(r) ) return r; if( imports.i == NULL ) { fprintf(stderr, "No definition for '%s' found in '%s/incoming'!\n", name, global.confdir); return RET_ERROR_MISSING; } r = incoming_prepare(imports.i); if( RET_WAS_ERROR(r) ) { incoming_free(imports.i); return r; } *result = imports.i; return r; } struct candidate { /* from candidate_read */ int ofs; char *control; struct signatures *signatures; /* from candidate_parse */ char *source, *sourceversion, *changesversion; struct strlist distributions, architectures, binaries; bool isbinNMU; struct candidate_file { /* set by _addfileline */ struct candidate_file *next; int ofs; /* to basename in struct incoming->files */ filetype type; /* all NULL if it is the .changes itself, * otherwise the data from the .changes for this file: */ char *section; char *priority; architecture_t architecture_atom; char *name; /* like above, but updated once files are copied */ struct checksums *checksums; /* set later */ bool used; char *tempfilename; /* distribution-unspecific contents of the packages */ /* - only for FE_BINARY types: */ struct deb_headers deb; /* - only for fe_DSC types */ struct dsc_headers dsc; /* only valid while parsing */ struct hashes h; } *files; struct candidate_perdistribution { struct candidate_perdistribution *next; struct distribution *into; bool skip; struct candidate_package { /* a package is something installing files, including * the pseudo-package for the .changes file, if that is * to be included */ struct candidate_package *next; const struct candidate_file *master; component_t component_atom; packagetype_t packagetype; struct strlist filekeys; /* a list of pointers to the files belonging to those * filekeys, NULL if it does not need linking/copying */ const struct candidate_file **files; /* only for FE_PACKAGE: */ char *control; /* only for fe_DSC */ char *directory; /* true if skipped because already there or newer */ bool skip; } *packages; } *perdistribution; /* the logsubdir, and the list of files to put there, otherwise both NULL */ char *logsubdir; int logcount; const struct candidate_file **logfiles; }; static void candidate_file_free(/*@only@*/struct candidate_file *f) { checksums_free(f->checksums); free(f->section); free(f->priority); free(f->name); if( FE_BINARY(f->type) ) binaries_debdone(&f->deb); if( f->type == fe_DSC ) sources_done(&f->dsc); if( f->tempfilename != NULL ) { (void)unlink(f->tempfilename); free(f->tempfilename); f->tempfilename = NULL; } free(f); } static void candidate_package_free(/*@only@*/struct candidate_package *p) { free(p->control); free(p->directory); strlist_done(&p->filekeys); free(p->files); free(p); } static void candidate_free(/*@only@*/struct candidate *c) { if( c == NULL ) return; free(c->control); signatures_free(c->signatures); free(c->source); free(c->sourceversion); free(c->changesversion); strlist_done(&c->distributions); strlist_done(&c->architectures); strlist_done(&c->binaries); while( c->perdistribution != NULL ) { struct candidate_perdistribution *d = c->perdistribution; c->perdistribution = d->next; while( d->packages != NULL ) { struct candidate_package *p = d->packages; d->packages = p->next; candidate_package_free(p); } free(d); } while( c->files != NULL ) { struct candidate_file *f = c->files; c->files = f->next; candidate_file_free(f); } free(c->logsubdir); free(c->logfiles); free(c); } static retvalue candidate_newdistribution(struct candidate *c, struct distribution *distribution) { struct candidate_perdistribution *n,**pp = &c->perdistribution; while( *pp != NULL ) { if( (*pp)->into == distribution ) return RET_NOTHING; pp = &(*pp)->next; } n = calloc(1, sizeof(struct candidate_perdistribution)); if( n == NULL ) return RET_ERROR_OOM; n->into = distribution; *pp = n; return RET_OK; } static struct candidate_package *candidate_newpackage(struct candidate_perdistribution *fordistribution, const struct candidate_file *master) { struct candidate_package *n,**pp = &fordistribution->packages; while( *pp != NULL ) pp = &(*pp)->next; n = calloc(1, sizeof(struct candidate_package)); if( n != NULL ) { n->component_atom = atom_unknown; n->packagetype = atom_unknown; n->master = master; *pp = n; } return n; } static retvalue candidate_usefile(const struct incoming *i,const struct candidate *c,struct candidate_file *file); static retvalue candidate_read(struct incoming *i, int ofs, struct candidate **result, bool *broken) { struct candidate *n; retvalue r; n = calloc(1,sizeof(struct candidate)); if( n == NULL ) return RET_ERROR_OOM; n->ofs = ofs; /* first file of any .changes file is the file itself */ n->files = calloc(1,sizeof(struct candidate_file)); if( n->files == NULL ) { free(n); return RET_ERROR_OOM; } n->files->ofs = n->ofs; n->files->type = fe_CHANGES; r = candidate_usefile(i, n, n->files); assert( r != RET_NOTHING ); if( RET_WAS_ERROR(r) ) { candidate_free(n); return r; } assert( n->files->tempfilename != NULL ); r = signature_readsignedchunk(n->files->tempfilename, BASENAME(i,ofs), &n->control, &n->signatures, broken); assert( r != RET_NOTHING ); if( RET_WAS_ERROR(r) ) { candidate_free(n); return r; } *result = n; return RET_OK; } static retvalue candidate_addfileline(struct incoming *i, struct candidate *c, const char *fileline) { struct candidate_file **p, *n; char *basefilename; retvalue r; n = calloc(1,sizeof(struct candidate_file)); if( n == NULL ) return RET_ERROR_OOM; r = changes_parsefileline(fileline, &n->type, &basefilename, &n->h.hashes[cs_md5sum], &n->h.hashes[cs_length], &n->section, &n->priority, &n->architecture_atom, &n->name); if( RET_WAS_ERROR(r) ) { free(n); return r; } n->ofs = strlist_ofs(&i->files, basefilename); if( n->ofs < 0 ) { fprintf(stderr, "In '%s': file '%s' not found in the incoming dir!\n", i->files.values[c->ofs], basefilename); free(basefilename); candidate_file_free(n); return RET_ERROR_MISSING; } free(basefilename); p = &c->files; while( *p != NULL ) p = &(*p)->next; *p = n; return RET_OK; } static retvalue candidate_addhashes(struct incoming *i, struct candidate *c, enum checksumtype cs, const struct strlist *lines) { int j; for( j = 0 ; j < lines->count ; j++ ) { const char *fileline = lines->values[j]; struct candidate_file *f; const char *basefilename; struct hash_data hash, size; retvalue r; r = hashline_parse(BASENAME(i, c->ofs), fileline, cs, &basefilename, &hash, &size); if( !RET_IS_OK(r) ) return r; f = c->files; while( f != NULL && strcmp(BASENAME(i, f->ofs), basefilename) != 0 ) f = f->next; if( f == NULL ) { fprintf(stderr, "Warning: Ignoring file '%s' listed in '%s' but not in '%s' of '%s'!\n", basefilename, changes_checksum_names[cs], changes_checksum_names[cs_md5sum], BASENAME(i, c->ofs)); continue; } if( f->h.hashes[cs_length].len != size.len || memcmp(f->h.hashes[cs_length].start, size.start, size.len) != 0 ) { fprintf(stderr, "Error: Different size of '%s' listed in '%s' and '%s' of '%s'!\n", basefilename, changes_checksum_names[cs], changes_checksum_names[cs_md5sum], BASENAME(i, c->ofs)); return RET_ERROR; } f->h.hashes[cs] = hash; } return RET_OK; } static retvalue candidate_finalizechecksums(struct candidate *c) { struct candidate_file *f; retvalue r; /* store collected hashes as checksums structs, * starting after .changes file: */ for( f = c->files->next ; f != NULL ; f = f->next ) { r = checksums_initialize(&f->checksums, f->h.hashes); if( RET_WAS_ERROR(r) ) return r; } return RET_OK; } static retvalue candidate_parse(struct incoming *i, struct candidate *c) { retvalue r; struct strlist filelines[cs_hashCOUNT]; enum checksumtype cs; int j; #define R if( RET_WAS_ERROR(r) ) return r; #define E(err,...) { \ if( r == RET_NOTHING ) { \ fprintf(stderr, "In '%s': " err "\n",BASENAME(i,c->ofs), ## __VA_ARGS__ ); \ r = RET_ERROR; \ } \ if( RET_WAS_ERROR(r) ) return r; \ } r = chunk_getnameandversion(c->control, "Source", &c->source, &c->sourceversion); E("Missing 'Source' field!"); r = propersourcename(c->source); E("Malforce Source name!"); if( c->sourceversion != NULL ) { r = properversion(c->sourceversion); E("Malforce Source Version number!"); } r = chunk_getwordlist(c->control,"Binary",&c->binaries); E("Missing 'Binary' field!"); r = chunk_getwordlist(c->control,"Architecture",&c->architectures); E("Missing 'Architecture' field!"); r = chunk_getvalue(c->control,"Version",&c->changesversion); E("Missing 'Version' field!"); r = properversion(c->changesversion); E("Malforce Version number!"); // TODO: logic to detect binNMUs to warn against sources? if( c->sourceversion == NULL ) { c->sourceversion = strdup(c->changesversion); if( c->sourceversion == NULL ) return RET_ERROR_OOM; c->isbinNMU = false; } else { int cmp; r = dpkgversions_cmp(c->sourceversion, c->changesversion, &cmp); R; c->isbinNMU = cmp != 0; } r = chunk_getwordlist(c->control,"Distribution",&c->distributions); E("Missing 'Distribution' field!"); r = chunk_getextralinelist(c->control, changes_checksum_names[cs_md5sum], &filelines[cs_md5sum]); E("Missing '%s' field!", changes_checksum_names[cs_md5sum]); for( j = 0 ; j < filelines[cs_md5sum].count ; j++ ) { r = candidate_addfileline(i, c, filelines[cs_md5sum].values[j]); if( RET_WAS_ERROR(r) ) { strlist_done(&filelines[cs_md5sum]); return r; } } for( cs = cs_firstEXTENDED ; cs < cs_hashCOUNT ; cs++ ) { r = chunk_getextralinelist(c->control, changes_checksum_names[cs], &filelines[cs]); if( RET_IS_OK(r) ) r = candidate_addhashes(i, c, cs, &filelines[cs]); else strlist_init(&filelines[cs]); if( RET_WAS_ERROR(r) ) { while( cs-- > cs_md5sum ) strlist_done(&filelines[cs]); return r; } } r = candidate_finalizechecksums(c); for( cs = cs_md5sum ; cs < cs_hashCOUNT ; cs++ ) strlist_done(&filelines[cs]); R; if( c->files == NULL || c->files->next == NULL ) { fprintf(stderr,"In '%s': Empty 'Files' section!\n", BASENAME(i,c->ofs)); return RET_ERROR; } return RET_OK; } static retvalue candidate_earlychecks(struct incoming *i, struct candidate *c) { struct candidate_file *file; retvalue r; // TODO: allow being more permissive, // that will need some more checks later, though r = propersourcename(c->source); if( RET_WAS_ERROR(r) ) return r; r = properversion(c->sourceversion); if( RET_WAS_ERROR(r) ) return r; for( file = c->files ; file != NULL ; file = file->next ) { if( file->type != fe_CHANGES && file->type != fe_BYHAND && file->type != fe_LOG && !atom_defined(file->architecture_atom) ) { fprintf(stderr, "'%s' contains '%s' not matching an valid architecture in any distribution known!\n", BASENAME(i, c->ofs), BASENAME(i, file->ofs)); return RET_ERROR; } if( !FE_PACKAGE(file->type) ) continue; assert( atom_defined(file->architecture_atom) ); if( strlist_in(&c->architectures, atoms_architectures[file->architecture_atom]) ) continue; fprintf(stderr, "'%s' is not listed in the Architecture header of '%s' but file '%s' looks like it!\n", atoms_architectures[file->architecture_atom], BASENAME(i,c->ofs), BASENAME(i,file->ofs)); return RET_ERROR; } return RET_OK; } /* Is used before any other candidate fields are set */ static retvalue candidate_usefile(const struct incoming *i,const struct candidate *c,struct candidate_file *file) { const char *basefilename; char *origfile,*tempfilename; struct checksums *readchecksums; retvalue r; bool improves; const char *p; if( file->used && file->tempfilename != NULL ) return RET_OK; assert(file->tempfilename == NULL); basefilename = BASENAME(i, file->ofs); for( p = basefilename; *p != '\0' ; p++ ) { if( (0x80 & *(const unsigned char *)p) != 0 ) { fprintf(stderr, "Invalid filename '%s' listed in '%s': contains 8-bit characters\n", basefilename, BASENAME(i, c->ofs)); return RET_ERROR; } } tempfilename = calc_dirconcat(i->tempdir, basefilename); if( tempfilename == NULL ) return RET_ERROR_OOM; origfile = calc_dirconcat(i->directory, basefilename); if( origfile == NULL ) { free(tempfilename); return RET_ERROR_OOM; } r = checksums_copyfile(tempfilename, origfile, true, &readchecksums); free(origfile); if( RET_WAS_ERROR(r) ) { free(tempfilename); return r; } if( file->checksums == NULL ) { file->checksums = readchecksums; file->tempfilename = tempfilename; file->used = true; return RET_OK; } if( !checksums_check(file->checksums, readchecksums, &improves) ) { fprintf(stderr, "ERROR: File '%s' does not match expectations:\n", basefilename); checksums_printdifferences(stderr, file->checksums, readchecksums); checksums_free(readchecksums); deletefile(tempfilename); free(tempfilename); return RET_ERROR_WRONG_MD5; } if( improves ) { r = checksums_combine(&file->checksums, readchecksums, NULL); if( RET_WAS_ERROR(r) ) { checksums_free(readchecksums); deletefile(tempfilename); free(tempfilename); return r; } } checksums_free(readchecksums); file->tempfilename = tempfilename; file->used = true; return RET_OK; } static inline retvalue getsectionprioritycomponent(const struct incoming *i, const struct candidate *c, const struct distribution *into, const struct candidate_file *file, const char *name, const struct overrideinfo *oinfo, /*@out@*/const char **section_p, /*@out@*/const char **priority_p, /*@out@*/component_t *component) { retvalue r; const char *section, *priority; section = override_get(oinfo, SECTION_FIELDNAME); if( section == NULL ) { // TODO: warn about disparities here? section = file->section; } if( section == NULL || strcmp(section,"-") == 0 ) { fprintf(stderr, "No section found for '%s' ('%s' in '%s')!\n", name, BASENAME(i,file->ofs), BASENAME(i,c->ofs)); return RET_ERROR; } priority = override_get(oinfo, PRIORITY_FIELDNAME); if( priority == NULL ) { // TODO: warn about disparities here? priority = file->priority; } if( priority == NULL || strcmp(priority,"-") == 0 ) { fprintf(stderr, "No priority found for '%s' ('%s' in '%s')!\n", name, BASENAME(i,file->ofs), BASENAME(i,c->ofs)); return RET_ERROR; } r = guess_component(into->codename, &into->components, BASENAME(i,file->ofs), section, atom_unknown, component); if( RET_WAS_ERROR(r) ) { return r; } *section_p = section; *priority_p = priority; return RET_OK; } static retvalue candidate_read_deb(struct incoming *i,struct candidate *c,struct candidate_file *file) { retvalue r; r = binaries_readdeb(&file->deb, file->tempfilename, true); if( RET_WAS_ERROR(r) ) return r; if( strcmp(file->name, file->deb.name) != 0 ) { // TODO: add permissive thing to ignore this fprintf(stderr, "Name part of filename ('%s') and name within the file ('%s') do not match for '%s' in '%s'!\n", file->name, file->deb.name, BASENAME(i,file->ofs), BASENAME(i,c->ofs)); return RET_ERROR; } if( file->architecture_atom != file->deb.architecture_atom ) { // TODO: add permissive thing to ignore this in some cases // but do not forget to look into into->architectures then fprintf(stderr, "Architecture '%s' of '%s' does not match '%s' specified in '%s'!\n", atoms_architectures[file->deb.architecture_atom], BASENAME(i,file->ofs), atoms_architectures[file->architecture_atom], BASENAME(i,c->ofs)); return RET_ERROR; } if( strcmp(c->source, file->deb.source) != 0 ) { // TODO: add permissive thing to ignore this // (beware if tracking is active) fprintf(stderr, "Source header '%s' of '%s' and source name '%s' within the file '%s' do not match!\n", c->source, BASENAME(i,c->ofs), file->deb.source, BASENAME(i,file->ofs)); return RET_ERROR; } if( strcmp(c->sourceversion, file->deb.sourceversion) != 0 ) { // TODO: add permissive thing to ignore this // (beware if tracking is active) fprintf(stderr, "Source version '%s' of '%s' and source version '%s' within the file '%s' do not match!\n", c->sourceversion, BASENAME(i,c->ofs), file->deb.sourceversion, BASENAME(i,file->ofs)); return RET_ERROR; } if( ! strlist_in(&c->binaries, file->deb.name) ) { fprintf(stderr, "Name '%s' of binary '%s' is not listed in Binaries header of '%s'!\n", file->deb.name, BASENAME(i,file->ofs), BASENAME(i,c->ofs)); return RET_ERROR; } r = properpackagename(file->deb.name); if( RET_IS_OK(r) ) r = propersourcename(file->deb.source); if( RET_IS_OK(r) ) r = properversion(file->deb.version); if( RET_WAS_ERROR(r) ) return r; return RET_OK; } static retvalue candidate_read_dsc(struct incoming *i, struct candidate_file *file) { retvalue r; bool broken = false; char *p; r = sources_readdsc(&file->dsc, file->tempfilename, BASENAME(i, file->ofs), &broken); if( RET_WAS_ERROR(r) ) return r; p = calc_source_basename(file->dsc.name, file->dsc.version); if( p == NULL ) return RET_ERROR_OOM; r = checksumsarray_include(&file->dsc.files, p, file->checksums); if( RET_WAS_ERROR(r) ) { return r; } // TODO: take a look at "broken"... return RET_OK; } static retvalue candidate_read_files(struct incoming *i, struct candidate *c) { struct candidate_file *file; retvalue r; for( file = c->files ; file != NULL ; file = file->next ) { if( !FE_PACKAGE(file->type) ) continue; r = candidate_usefile(i, c, file); if( RET_WAS_ERROR(r) ) return r; assert(file->tempfilename != NULL); if( FE_BINARY(file->type) ) r = candidate_read_deb(i, c, file); else if( file->type == fe_DSC ) r = candidate_read_dsc(i, file); else { r = RET_ERROR; assert( FE_BINARY(file->type) || file->type == fe_DSC ); } if( RET_WAS_ERROR(r) ) return r; } return RET_OK; } static retvalue candidate_preparechangesfile(struct database *database,const struct candidate *c,struct candidate_perdistribution *per) { retvalue r; char *basefilename, *filekey; struct candidate_package *package; struct candidate_file *file; component_t component = component_strange; assert( c->files != NULL && c->files->ofs == c->ofs ); /* search for a component to use */ for( package = per->packages ; package != NULL ; package = package->next ) { if( atom_defined(package->component_atom) ) { component = package->component_atom; break; } } file = changesfile(c); /* make sure the file is already copied */ assert( file->used ); assert( file->checksums != NULL ); /* pseudo package containing the .changes file */ package = candidate_newpackage(per, c->files); if( package == NULL ) return RET_ERROR_OOM; basefilename = calc_changes_basename(c->source, c->changesversion, &c->architectures); if( basefilename == NULL ) return RET_ERROR_OOM; filekey = calc_filekey(component, c->source, basefilename); free(basefilename); if( filekey == NULL ) return RET_ERROR_OOM; r = strlist_init_singleton(filekey, &package->filekeys); if( RET_WAS_ERROR(r) ) return r; assert( package->filekeys.count == 1 ); filekey = package->filekeys.values[0]; package->files = calloc(1, sizeof(struct candidate_file *)); if( package->files == NULL ) return RET_ERROR_OOM; r = files_canadd(database, filekey, file->checksums); if( RET_WAS_ERROR(r) ) return r; if( RET_IS_OK(r) ) package->files[0] = file; return RET_OK; } static retvalue prepare_deb(struct database *database,const struct incoming *i,const struct candidate *c,struct candidate_perdistribution *per,const struct candidate_file *file) { const char *section IFSTUPIDCC(=NULL), *priority IFSTUPIDCC(=NULL), *filekey; const struct overrideinfo *oinfo; struct candidate_package *package; const struct distribution *into = per->into; retvalue r; assert( FE_BINARY(file->type) ); assert( file->tempfilename != NULL ); assert( file->deb.name != NULL ); package = candidate_newpackage(per, file); if( package == NULL ) return RET_ERROR_OOM; assert( file == package->master ); if( file->type == fe_DEB ) package->packagetype = pt_deb; else package->packagetype = pt_udeb; oinfo = override_search(file->type==fe_UDEB?into->overrides.udeb: into->overrides.deb, file->name); r = getsectionprioritycomponent(i,c,into,file, file->name, oinfo, §ion, &priority, &package->component_atom); if( RET_WAS_ERROR(r) ) return r; if( file->type == fe_UDEB && !atomlist_in(&into->udebcomponents, package->component_atom)) { fprintf(stderr, "Cannot put file '%s' of '%s' into component '%s',\n" "as it is not listed in UDebComponents of '%s'!\n", BASENAME(i,file->ofs), BASENAME(i,c->ofs), atoms_components[package->component_atom], into->codename); return RET_ERROR; } r = binaries_calcfilekeys(package->component_atom, &file->deb, package->packagetype, &package->filekeys); if( RET_WAS_ERROR(r) ) return r; assert( package->filekeys.count == 1 ); filekey = package->filekeys.values[0]; package->files = calloc(1, sizeof(struct candidate_file *)); if( package->files == NULL ) return RET_ERROR_OOM; r = files_canadd(database, filekey, file->checksums); if( RET_WAS_ERROR(r) ) return r; if( RET_IS_OK(r) ) package->files[0] = file; r = binaries_complete(&file->deb, filekey, file->checksums, oinfo, section, priority, &package->control); if( RET_WAS_ERROR(r) ) return r; return RET_OK; } static retvalue prepare_source_file(struct database *database, const struct incoming *i, const struct candidate *c, const char *filekey, const char *basefilename, struct checksums **checksums_p, int package_ofs, /*@out@*/const struct candidate_file **foundfile_p){ struct candidate_file *f; const struct checksums * const checksums = *checksums_p; retvalue r; bool improves; f = c->files; while( f != NULL && (f->checksums == NULL || strcmp(BASENAME(i, f->ofs), basefilename) != 0) ) f = f->next; if( f == NULL ) { r = files_canadd(database, filekey, checksums); if( !RET_IS_OK(r) ) return r; /* no file by this name and also no file with these * characteristics in the pool, look for differently-named * file with the same characteristics: */ f = c->files; while( f != NULL && ( f->checksums == NULL || !checksums_check(f->checksums, checksums, NULL))) f = f->next; if( f == NULL ) { fprintf(stderr, "file '%s' is needed for '%s', not yet registered in the pool and not found in '%s'\n", basefilename, BASENAME(i, package_ofs), BASENAME(i, c->ofs)); return RET_ERROR; } /* otherwise proceed with the found file: */ } if( !checksums_check(f->checksums, checksums, &improves) ) { fprintf(stderr, "file '%s' has conflicting checksums listed in '%s' and '%s'!\n", basefilename, BASENAME(i, c->ofs), BASENAME(i, package_ofs)); return RET_ERROR; } if( improves ) { /* put additional checksums from the .dsc to the information * found in .changes, so that a file matching those in .changes * but not in .dsc is detected */ r = checksums_combine(&f->checksums, checksums, NULL); assert( r != RET_NOTHING ); if( RET_WAS_ERROR(r) ) return r; } r = files_canadd(database, filekey, f->checksums); if( r == RET_NOTHING ) { /* already in the pool, mark as used (in the sense * of "only not needed because it is already there") */ f->used = true; } else if( RET_IS_OK(r) ) { /* don't have this file in the pool, make sure it is ready * here */ r = candidate_usefile(i, c, f); if( RET_WAS_ERROR(r) ) return r; // TODO: update checksums to now received checksums? *foundfile_p = f; } if( !RET_WAS_ERROR(r) && !checksums_iscomplete(checksums) ) { /* update checksums so the source index can show them */ r = checksums_combine(checksums_p, f->checksums, NULL); } return r; } static retvalue prepare_dsc(struct database *database,const struct incoming *i,const struct candidate *c,struct candidate_perdistribution *per,const struct candidate_file *file) { const char *section IFSTUPIDCC(=NULL), *priority IFSTUPIDCC(=NULL); const struct overrideinfo *oinfo; struct candidate_package *package; const struct distribution *into = per->into; retvalue r; int j; assert( file->type == fe_DSC ); assert( file->tempfilename != NULL ); assert( file->dsc.name != NULL ); package = candidate_newpackage(per, file); if( package == NULL ) return RET_ERROR_OOM; assert( file == package->master ); package->packagetype = pt_dsc; if( c->isbinNMU ) { // TODO: add permissive thing to ignore this fprintf(stderr, "Source package ('%s') in '%s', which look like a binNMU (as '%s' and '%s' differ)!\n", BASENAME(i,file->ofs), BASENAME(i,c->ofs), c->sourceversion, c->changesversion); return RET_ERROR; } if( strcmp(file->name, file->dsc.name) != 0 ) { // TODO: add permissive thing to ignore this fprintf(stderr, "Name part of filename ('%s') and name within the file ('%s') do not match for '%s' in '%s'!\n", file->name, file->dsc.name, BASENAME(i,file->ofs), BASENAME(i,c->ofs)); return RET_ERROR; } if( strcmp(c->source, file->dsc.name) != 0 ) { // TODO: add permissive thing to ignore this // (beware if tracking is active) fprintf(stderr, "Source header '%s' of '%s' and name '%s' within the file '%s' do not match!\n", c->source, BASENAME(i,c->ofs), file->dsc.name, BASENAME(i,file->ofs)); return RET_ERROR; } if( strcmp(c->sourceversion, file->dsc.version) != 0 ) { // TODO: add permissive thing to ignore this // (beware if tracking is active) fprintf(stderr, "Source version '%s' of '%s' and version '%s' within the file '%s' do not match!\n", c->sourceversion, BASENAME(i,c->ofs), file->dsc.version, BASENAME(i,file->ofs)); return RET_ERROR; } r = propersourcename(file->dsc.name); if( RET_IS_OK(r) ) r = properversion(file->dsc.version); if( RET_IS_OK(r) ) r = properfilenames(&file->dsc.files.names); if( RET_WAS_ERROR(r) ) return r; oinfo = override_search(into->overrides.dsc, file->dsc.name); r = getsectionprioritycomponent(i, c, into, file, file->dsc.name, oinfo, §ion, &priority, &package->component_atom); if( RET_WAS_ERROR(r) ) return r; package->directory = calc_sourcedir(package->component_atom, file->dsc.name); if( package->directory == NULL ) return RET_ERROR_OOM; r = calc_dirconcats(package->directory, &file->dsc.files.names, &package->filekeys); if( RET_WAS_ERROR(r) ) return r; package->files = calloc(package->filekeys.count,sizeof(struct candidate *)); if( package->files == NULL ) return RET_ERROR_OOM; r = files_canadd(database, package->filekeys.values[0], file->checksums); if( RET_IS_OK(r) ) package->files[0] = file; if( RET_WAS_ERROR(r) ) return r; for( j = 1 ; j < package->filekeys.count ; j++ ) { r = prepare_source_file(database, i, c, package->filekeys.values[j], file->dsc.files.names.values[j], &file->dsc.files.checksums[j], file->ofs, &package->files[j]); if( RET_WAS_ERROR(r) ) return r; } r = sources_complete(&file->dsc, package->directory, oinfo, section, priority, &package->control); if( RET_WAS_ERROR(r) ) return r; return RET_OK; } static retvalue candidate_preparebyhands(struct database *database, const struct incoming *i, const struct candidate *c, struct candidate_perdistribution *per) { retvalue r; char *byhanddir; struct candidate_package *package; struct candidate_file *firstbyhand = NULL, *file; component_t component = component_strange; int count = 0; for( file = c->files ; file != NULL ; file = file->next ) { if( file->type == fe_BYHAND ) { count++; if( firstbyhand == NULL ) firstbyhand = file; } } if( count == 0 ) return RET_NOTHING; /* search for a component to use */ for( package = per->packages ; package != NULL ; package = package->next ) { if( atom_defined(package->component_atom) ) { component = package->component_atom; break; } } /* pseudo package containing byhand files */ package = candidate_newpackage(per, firstbyhand); if( FAILEDTOALLOC(package) ) return RET_ERROR_OOM; r = strlist_init_n(count, &package->filekeys); if( RET_WAS_ERROR(r) ) return r; package->files = calloc(count, sizeof(struct candidate_file *)); if( FAILEDTOALLOC(package->files) ) return RET_ERROR_OOM; byhanddir = calc_byhanddir(component, c->source, c->changesversion); if( FAILEDTOALLOC(byhanddir) ) return RET_ERROR_OOM; for( file = c->files ; file != NULL ; file = file->next ) { char *filekey; if( file->type != fe_BYHAND ) continue; r = candidate_usefile(i, c, file); if( RET_WAS_ERROR(r) ) { free(byhanddir); return r; } filekey = calc_dirconcat(byhanddir, BASENAME(i,file->ofs)); if( FAILEDTOALLOC(filekey) ) { free(byhanddir); return RET_ERROR_OOM; } r = files_canadd(database, filekey, file->checksums); if( RET_WAS_ERROR(r) ) { free(byhanddir); return r; } if( RET_IS_OK(r) ) package->files[package->filekeys.count] = file; r = strlist_add(&package->filekeys, filekey); assert( r == RET_OK ); } free(byhanddir); assert( package->filekeys.count == count ); return RET_OK; } static retvalue candidate_preparelogs(struct database *database, const struct incoming *i, const struct candidate *c, struct candidate_perdistribution *per) { retvalue r; struct candidate_package *package; struct candidate_file *firstlog = NULL, *file; component_t component = component_strange; int count = 0; for( file = c->files ; file != NULL ; file = file->next ) { if( file->type == fe_LOG ) { count++; if( firstlog == NULL ) firstlog = file; } } if( count == 0 ) return RET_NOTHING; /* search for a component to use */ for( package = per->packages ; package != NULL ; package = package->next ) { if( atom_defined(package->component_atom) ) { component = package->component_atom; break; } } /* pseudo package containing log files */ package = candidate_newpackage(per, firstlog); if( FAILEDTOALLOC(package) ) return RET_ERROR_OOM; r = strlist_init_n(count, &package->filekeys); if( RET_WAS_ERROR(r) ) return r; package->files = calloc(count, sizeof(struct candidate_file *)); if( FAILEDTOALLOC(package->files) ) return RET_ERROR_OOM; for( file = c->files ; file != NULL ; file = file->next ) { char *filekey; if( file->type != fe_LOG ) continue; r = candidate_usefile(i, c, file); if( RET_WAS_ERROR(r) ) return r; // TODO: add same checks on the basename contents? filekey = calc_filekey(component, c->source, BASENAME(i,file->ofs)); if( FAILEDTOALLOC(filekey) ) return RET_ERROR_OOM; r = files_canadd(database, filekey, file->checksums); if( RET_WAS_ERROR(r) ) return r; if( RET_IS_OK(r) ) package->files[package->filekeys.count] = file; r = strlist_add(&package->filekeys, filekey); assert( r == RET_OK ); } assert( package->filekeys.count == count ); return RET_OK; } static retvalue prepare_for_distribution(struct database *database,const struct incoming *i,const struct candidate *c,struct candidate_perdistribution *d) { struct candidate_file *file; retvalue r; d->into->lookedat = true; for( file = c->files ; file != NULL ; file = file->next ) { switch( file->type ) { case fe_UDEB: case fe_DEB: r = prepare_deb(database, i, c, d, file); break; case fe_DSC: r = prepare_dsc(database, i, c, d, file); break; default: r = RET_NOTHING; break; } if( RET_WAS_ERROR(r) ) { return r; } } if( d->into->tracking != dt_NONE ) { if( d->into->trackingoptions.includebyhand ) { r = candidate_preparebyhands(database, i, c, d); if( RET_WAS_ERROR(r) ) return r; } if( d->into->trackingoptions.includelogs ) { r = candidate_preparelogs(database, i, c, d); if( RET_WAS_ERROR(r) ) return r; } if( d->into->trackingoptions.includechanges ) { r = candidate_preparechangesfile(database, c, d); if( RET_WAS_ERROR(r) ) return r; } } //... check if something would be done ... return RET_OK; } static retvalue candidate_addfiles(struct database *database,struct candidate *c) { int j; struct candidate_perdistribution *d; struct candidate_package *p; retvalue r; for( d = c->perdistribution ; d != NULL ; d = d->next ) { for( p = d->packages ; p != NULL ; p = p->next ) { if( p->skip ) continue; for( j = 0 ; j < p->filekeys.count ; j++ ) { const struct candidate_file *f = p->files[j]; if( f == NULL ) continue; assert(f->tempfilename != NULL); r = files_hardlinkandadd(database, f->tempfilename, p->filekeys.values[j], f->checksums); if( RET_WAS_ERROR(r) ) return r; } } } return RET_OK; } static retvalue add_dsc(struct database *database, struct distribution *into, struct trackingdata *trackingdata, struct candidate_package *p) { retvalue r; struct target *t = distribution_getpart(into, p->component_atom, architecture_source, pt_dsc); assert( logger_isprepared(into->logger) ); /* finally put it into the source distribution */ r = target_initpackagesdb(t, database, READWRITE); if( !RET_WAS_ERROR(r) ) { retvalue r2; if( interrupted() ) r = RET_ERROR_INTERRUPTED; else r = target_addpackage(t, into->logger, database, p->master->dsc.name, p->master->dsc.version, p->control, &p->filekeys, false, trackingdata, architecture_source, NULL, NULL); r2 = target_closepackagesdb(t); RET_ENDUPDATE(r,r2); } RET_UPDATE(into->status, r); return r; } static retvalue checkadd_dsc(struct database *database, struct distribution *into, const struct incoming *i, bool tracking, struct candidate_package *p) { retvalue r; struct target *t = distribution_getpart(into, p->component_atom, architecture_source, pt_dsc); /* check for possible errors putting it into the source distribution */ r = target_initpackagesdb(t, database, READONLY); if( !RET_WAS_ERROR(r) ) { retvalue r2; if( interrupted() ) r = RET_ERROR_INTERRUPTED; else r = target_checkaddpackage(t, p->master->dsc.name, p->master->dsc.version, tracking, i->permit[pmf_oldpackagenewer]); r2 = target_closepackagesdb(t); RET_ENDUPDATE(r,r2); } return r; } static retvalue candidate_add_into(struct database *database, const struct incoming *i, const struct candidate *c, const struct candidate_perdistribution *d) { retvalue r; struct candidate_package *p; struct trackingdata trackingdata; struct distribution *into = d->into; trackingdb tracks; const char *changesfilekey = NULL; char *origfilename; struct atomlist binary_architectures; if( interrupted() ) return RET_ERROR_INTERRUPTED; into->lookedat = true; if( into->logger != NULL ) { r = logger_prepare(d->into->logger); if( RET_WAS_ERROR(r) ) return r; } tracks = NULL; if( into->tracking != dt_NONE ) { r = tracking_initialize(&tracks, database, into, false); if( RET_WAS_ERROR(r) ) return r; } if( tracks != NULL ) { r = trackingdata_summon(tracks, c->source, c->sourceversion, &trackingdata); if( RET_WAS_ERROR(r) ) { (void)tracking_done(tracks); return r; } if( into->trackingoptions.needsources ) { // TODO, but better before we start adding... } } origfilename = calc_dirconcat(i->directory, BASENAME(i, changesfile(c)->ofs)); causingfile = origfilename; atomlist_init(&binary_architectures); for( p = d->packages ; p != NULL ; p = p->next ) { if( FE_BINARY(p->master->type) ) { architecture_t a = p->master->architecture_atom; if( a != architecture_all ) atomlist_add_uniq(&binary_architectures, a); } } r = RET_OK; for( p = d->packages ; p != NULL ; p = p->next ) { if( p->skip ) { if( verbose >= 0 ) printf( "Not putting '%s' in '%s' as already in there with equal or newer version.\n", BASENAME(i,p->master->ofs), into->codename); continue; } if( p->master->type == fe_DSC ) { r = add_dsc(database, into, (tracks==NULL)?NULL:&trackingdata, p); } else if( FE_BINARY(p->master->type) ) { architecture_t a = p->master->architecture_atom; const struct atomlist *as, architectures = {&a, 1, 1}; if( i->options[iof_limit_arch_all] && a == architecture_all && binary_architectures.count > 0 ) as = &binary_architectures; else as = &architectures; r = binaries_adddeb(&p->master->deb, database, as, p->packagetype, into, (tracks==NULL)?NULL:&trackingdata, p->component_atom, &p->filekeys, p->control); } else if( p->master->type == fe_CHANGES ) { /* finally add the .changes to tracking, if requested */ assert( p->master->name == NULL ); assert( tracks != NULL ); r = trackedpackage_adddupfilekeys(trackingdata.tracks, trackingdata.pkg, ft_CHANGES, &p->filekeys, false, database); if( p->filekeys.count > 0 ) changesfilekey = p->filekeys.values[0]; } else if( p->master->type == fe_BYHAND ) { assert( tracks != NULL ); r = trackedpackage_adddupfilekeys(trackingdata.tracks, trackingdata.pkg, ft_XTRA_DATA, &p->filekeys, false, database); } else if( p->master->type == fe_LOG ) { assert( tracks != NULL ); r = trackedpackage_adddupfilekeys(trackingdata.tracks, trackingdata.pkg, ft_LOG, &p->filekeys, false, database); } else r = RET_ERROR_INTERNAL; if( RET_WAS_ERROR(r) ) { // TODO: remove files not yet referenced break; } } atomlist_done(&binary_architectures); if( tracks != NULL ) { retvalue r2; r2 = trackingdata_finish(tracks, &trackingdata, database); RET_UPDATE(r,r2); r2 = tracking_done(tracks); RET_ENDUPDATE(r,r2); } if( RET_WAS_ERROR(r) ) { free(origfilename); return r; } logger_logchanges(into->logger, into->codename, c->source, c->changesversion, c->control, changesfile(c)->tempfilename, changesfilekey); free(origfilename); causingfile = NULL; return RET_OK; } static inline retvalue candidate_checkadd_into(struct database *database,const struct incoming *i,const struct candidate_perdistribution *d) { retvalue r; struct candidate_package *p; struct distribution *into = d->into; bool somethingtodo = false; for( p = d->packages ; p != NULL ; p = p->next ) { if( p->master->type == fe_DSC ) { r = checkadd_dsc(database, into, i, into->tracking != dt_NONE, p); } else if( FE_BINARY(p->master->type) ) { r = binaries_checkadddeb(&p->master->deb, database, p->master->architecture_atom, p->packagetype, into, into->tracking != dt_NONE, p->component_atom, i->permit[pmf_oldpackagenewer]); } else if( p->master->type == fe_CHANGES || p->master->type == fe_BYHAND || p->master->type == fe_LOG ) { continue; } else r = RET_ERROR_INTERNAL; if( RET_WAS_ERROR(r) ) return r; if( r == RET_NOTHING ) p->skip = true; else somethingtodo = true; } if( somethingtodo ) return RET_OK; else return RET_NOTHING; } static inline bool isallowed(UNUSED(struct incoming *i), struct candidate *c, UNUSED(struct distribution *into), struct upload_conditions *conditions) { const struct candidate_file *file; do switch( uploaders_nextcondition(conditions) ) { case uc_ACCEPTED: return true; case uc_REJECTED: return false; case uc_SOURCENAME: assert( c->source != NULL ); (void)uploaders_verifystring(conditions, c->source); break; case uc_SECTIONS: for( file = c->files ; file != NULL ; file = file->next ) { if( !FE_PACKAGE(file->type) ) continue; if( !uploaders_verifystring(conditions, (file->section == NULL) ?"-":file->section) ) break; } break; case uc_BINARIES: for( file = c->files ; file != NULL ; file = file->next ) { if( !FE_BINARY(file->type) ) continue; if( !uploaders_verifystring(conditions, file->name) ) break; } break; case uc_ARCHITECTURES: for( file = c->files ; file != NULL ; file = file->next ) { if( !FE_PACKAGE(file->type) ) continue; if( !uploaders_verifyatom(conditions, file->architecture_atom) ) break; } break; } while( true ); } static retvalue candidate_checkpermissions(struct incoming *i, struct candidate *c, struct distribution *into) { retvalue r; struct upload_conditions *conditions; bool allowed; /* no rules means allowed */ if( into->uploaders == NULL ) return RET_OK; r = distribution_loaduploaders(into); if( RET_WAS_ERROR(r) ) return r; assert(into->uploaderslist != NULL); r = uploaders_permissions(into->uploaderslist, c->signatures, &conditions); assert( r != RET_NOTHING ); if( RET_WAS_ERROR(r) ) return r; allowed = isallowed(i, c, into, conditions); free(conditions); if( allowed ) return RET_OK; else /* reject */ return RET_NOTHING; } static retvalue check_architecture_availability(const struct incoming *i, const struct candidate *c) { struct candidate_perdistribution *d; bool check_all_availability = false; bool have_all_available = false; int j; // TODO: switch to instead ensure every architecture can be put into // one distribution at least would be nice. If implementing this do not // forget to check later to only put files in when the distribution can // cope with that. for( j = 0 ; j < c->architectures.count ; j++ ) { const char *architecture = c->architectures.values[j]; if( strcmp(architecture, "all") == 0 ) { check_all_availability = true; continue; } for( d = c->perdistribution ; d != NULL ; d = d->next ) { if( atomlist_in(&d->into->architectures, architecture_find(architecture)) ) continue; fprintf(stderr, "'%s' lists architecture '%s' not found in distribution '%s'!\n", BASENAME(i,c->ofs), architecture, d->into->codename); return RET_ERROR; } if( strcmp(architecture, "source") != 0 ) have_all_available = true; } if( check_all_availability && ! have_all_available ) { for( d = c->perdistribution ; d != NULL ; d = d->next ) { if( d->into->architectures.count > 1 ) continue; if( d->into->architectures.count > 0 && d->into->architectures.atoms[0] != architecture_source) continue; fprintf(stderr, "'%s' lists architecture 'all' but no binary architecture found in distribution '%s'!\n", BASENAME(i,c->ofs), d->into->codename); return RET_ERROR; } } return RET_OK; } static retvalue create_uniq_logsubdir(const char *logdir, const char *name, const char *version, const struct strlist *architectures, /*@out@*/char **subdir_p) { char *dir, *p; size_t l; retvalue r; r = dirs_make_recursive(logdir); if( RET_WAS_ERROR(r) ) return r; p = calc_changes_basename(name, version, architectures); if( FAILEDTOALLOC(p) ) return RET_ERROR_OOM; dir = calc_dirconcat(logdir, p); free(p); if( FAILEDTOALLOC(dir) ) return RET_ERROR_OOM; l = strlen(dir); assert( l > 8 && strcmp(dir + l - 8 , ".changes") == 0 ); memset(dir + l - 7, '0', 7); r = dirs_create(dir); while( r == RET_NOTHING ) { p = dir + l - 1; while( *p == '9' ) { *p = '0'; p--; } if( *p < '0' || *p > '8' ) { fprintf(stderr, "Failed to create a new directory of the form '%s'\n" "it looks like all 10000000 such directories are already there...\n", dir); return RET_ERROR; } (*p)++; r = dirs_create(dir); } *subdir_p = dir; return RET_OK; } static retvalue candidate_prepare_logdir(struct incoming *i, struct candidate *c) { int count, j; struct candidate_file *file; retvalue r; r = create_uniq_logsubdir(i->logdir, c->source, c->changesversion, &c->architectures, &c->logsubdir); assert( RET_IS_OK(r) ); if( RET_WAS_ERROR(r) ) return RET_ERROR_OOM; count = 0; for( file = c->files ; file != NULL ; file = file->next ) { if( file->ofs == c->ofs || file->type == fe_LOG || (file->type == fe_BYHAND && !file->used) ) count++; } c->logcount = count; c->logfiles = calloc(count, sizeof(struct canidate_file*)); if( FAILEDTOALLOC(c->logfiles) ) return RET_ERROR_OOM; j = 0; for( file = c->files ; file != NULL ; file = file->next ) { if( file->ofs == c->ofs || file->type == fe_LOG || (file->type == fe_BYHAND && !file->used) ) { r = candidate_usefile(i, c, file); if( RET_WAS_ERROR(r) ) return r; c->logfiles[j++] = file; } } assert( count == j ); return RET_OK; } static retvalue candidate_finish_logdir(struct incoming *i, struct candidate *c) { int j; for( j = 0 ; j < c->logcount ; j++ ) { retvalue r; const struct candidate_file *f = c->logfiles[j]; r = checksums_hardlink(c->logsubdir, BASENAME(i, f->ofs), f->tempfilename, f->checksums); if( RET_WAS_ERROR(r) ) return r; } return RET_OK; } static retvalue candidate_add(struct database *database, struct incoming *i, struct candidate *c) { struct candidate_perdistribution *d; struct candidate_file *file; retvalue r; bool somethingtodo; assert( c->perdistribution != NULL ); /* check if every distribution this is to be added to supports * all architectures we have files for */ r = check_architecture_availability(i, c); if( RET_WAS_ERROR(r) ) return r; for( d = c->perdistribution ; d != NULL ; d = d->next ) { r = distribution_loadalloverrides(d->into); if( RET_WAS_ERROR(r) ) return r; } // TODO: once uploaderlist allows to look for package names or existing override // entries or such things, check package names here enable checking for content // name with outer name /* when we get here, the package is allowed in, now we have to * read the parts and check all stuff we only know now */ r = candidate_read_files(i, c); if( RET_WAS_ERROR(r) ) return r; /* now the distribution specific part starts: */ for( d = c->perdistribution ; d != NULL ; d = d->next ) { r = prepare_for_distribution(database, i, c, d); if( RET_WAS_ERROR(r) ) return r; } if( i->logdir != NULL ) { r = candidate_prepare_logdir(i, c); if( RET_WAS_ERROR(r) ) return r; } for( file = c->files ; file != NULL ; file = file->next ) { if( !file->used && !i->permit[pmf_unused_files] ) { // TODO: find some way to mail such errors... fprintf(stderr, "Error: '%s' contains unused file '%s'!\n" "(Do Permit: unused_files to conf/incoming to ignore and\n" " additionaly Cleanup: unused_files to delete them)\n", BASENAME(i,c->ofs), BASENAME(i,file->ofs)); if( file->type == fe_LOG || file->type == fe_BYHAND ) fprintf(stderr, "Alternatively, you can also add a LogDir: for '%s' into conf/incoming\n" "then files like that will be stored there.\n", i->name); return RET_ERROR; } } /* additional test run to see if anything could go wrong, * or if there are already newer versions */ somethingtodo = false; for( d = c->perdistribution ; d != NULL ; d = d->next ) { r = candidate_checkadd_into(database, i, d); if( RET_WAS_ERROR(r) ) return r; if( r == RET_NOTHING ) d->skip = true; else somethingtodo = true; } if( ! somethingtodo ) { if( verbose >= 0 ) { printf("Skipping %s because all packages are skipped!\n", BASENAME(i,c->ofs)); } for( file = c->files ; file != NULL ; file = file->next ) { if( file->used || i->cleanup[cuf_unused_files] ) i->delete[file->ofs] = true; } return RET_NOTHING; } // TODO: make sure not two different files are supposed to be installed // as the same filekey. /* the actual adding of packages, make sure what can be checked was * checked by now */ /* make hardlinks/copies of the files */ r = candidate_addfiles(database, c); if( RET_WAS_ERROR(r) ) return r; if( interrupted() ) return RET_ERROR_INTERRUPTED; if( i->logdir != NULL ) { r = candidate_finish_logdir(i, c); if( RET_WAS_ERROR(r) ) return r; } if( interrupted() ) return RET_ERROR_INTERRUPTED; r = RET_OK; for( d = c->perdistribution ; d != NULL ; d = d->next ) { if( d->skip ) continue; r = candidate_add_into(database, i, c, d); if( RET_WAS_ERROR(r) ) return r; } /* mark files as done */ for( file = c->files ; file != NULL ; file = file->next ) { if( file->used ) i->processed[file->ofs] = true; if( file->used || i->cleanup[cuf_unused_files] ) { i->delete[file->ofs] = true; } } return RET_OK; } static retvalue process_changes(struct database *database, struct incoming *i, int ofs) { struct candidate *c IFSTUPIDCC(=NULL); retvalue r; int j,k; bool broken = false, tried = false; r = candidate_read(i, ofs, &c, &broken); if( RET_WAS_ERROR(r) ) return r; assert( RET_IS_OK(r) ); r = candidate_parse(i, c); if( RET_WAS_ERROR(r) ) { candidate_free(c); return r; } r = candidate_earlychecks(i, c); if( RET_WAS_ERROR(r) ) { if( i->cleanup[cuf_on_error] ) { struct candidate_file *file; i->delete[c->ofs] = true; for( file = c->files ; file != NULL ; file = file->next ) { i->delete[file->ofs] = true; } } candidate_free(c); return r; } for( k = 0 ; k < c->distributions.count ; k++ ) { const char *name = c->distributions.values[k]; for( j = 0 ; j < i->allow.count ; j++ ) { // TODO: implement "*" if( strcmp(name, i->allow.values[j]) == 0 ) { tried = true; r = candidate_checkpermissions(i, c, i->allow_into[j]); if( r == RET_NOTHING ) continue; if( RET_IS_OK(r) ) r = candidate_newdistribution(c, i->allow_into[j]); if( RET_WAS_ERROR(r) ) { candidate_free(c); return r; } else break; } } if( c->perdistribution != NULL && !i->options[iof_multiple_distributions] ) break; } if( c->perdistribution == NULL && i->default_into != NULL ) { tried = true; r = candidate_checkpermissions(i, c, i->default_into); if( RET_WAS_ERROR(r) ) { candidate_free(c); return r; } if( RET_IS_OK(r) ) { r = candidate_newdistribution(c, i->default_into); } } if( c->perdistribution == NULL ) { fprintf(stderr, tried?"No distribution accepting '%s'!\n": "No distribution found for '%s'!\n", i->files.values[ofs]); if( i->cleanup[cuf_on_deny] ) { struct candidate_file *file; i->delete[c->ofs] = true; for( file = c->files ; file != NULL ; file = file->next ) { // TODO: implement same-owner check if( !i->cleanup[cuf_on_deny_check_owner] ) i->delete[file->ofs] = true; } } r = RET_ERROR_INCOMING_DENY; } else { if( broken ) { fprintf(stderr, "'%s' is signed with only invalid signatures.\n" "If this was not corruption but willfull modification,\n" "remove the signatures and try again.\n", i->files.values[ofs]); r = RET_ERROR; } else r = candidate_add(database, i, c); if( RET_WAS_ERROR(r) && i->cleanup[cuf_on_error] ) { struct candidate_file *file; i->delete[c->ofs] = true; for( file = c->files ; file != NULL ; file = file->next ) { i->delete[file->ofs] = true; } } } logger_wait(); candidate_free(c); return r; } static inline /*@null@*/char *create_uniq_subdir(const char *basedir) { char date[16], *dir; unsigned long number = 0; retvalue r; time_t curtime; struct tm *tm; int e; r = dirs_make_recursive(basedir); if( RET_WAS_ERROR(r) ) return NULL; if( time(&curtime) == (time_t)-1 ) tm = NULL; else tm = gmtime(&curtime); if( tm == NULL || strftime(date, 16, "%Y-%m-%d", tm) != 10 ) strcpy(date, "timeerror"); for( number = 0 ; number < 10000 ; number ++ ) { dir = mprintf("%s/%s-%lu", basedir, date, number); if( FAILEDTOALLOC(dir) ) return NULL; if( mkdir(dir, 0777) == 0 ) return dir; e = errno; if( e != EEXIST ) { fprintf(stderr, "Error %d creating directory '%s': %s\n", e, dir, strerror(e)); free(dir); return NULL; } free(dir); } fprintf(stderr, "Could not create unique subdir in '%s'!\n", basedir); return NULL; } /* tempdir should ideally be on the same partition like the pooldir */ retvalue process_incoming(struct database *database, struct distribution *distributions, const char *name, const char *changesfilename) { struct incoming *i; retvalue result,r; int j; char *morguedir; result = RET_NOTHING; r = incoming_init(distributions, name, &i); if( RET_WAS_ERROR(r) ) return r; for( j = 0 ; j < i->files.count ; j ++ ) { const char *basefilename = i->files.values[j]; size_t l = strlen(basefilename); #define C_SUFFIX ".changes" #define C_LEN strlen(C_SUFFIX) if( l <= C_LEN || strcmp(basefilename + (l - C_LEN ), C_SUFFIX) != 0 ) continue; if( changesfilename != NULL && strcmp(basefilename, changesfilename) != 0 ) continue; /* a .changes file, check it */ r = process_changes(database, i, j); RET_UPDATE(result, r); } logger_wait(); if( i->morguedir == NULL ) morguedir = NULL; else { morguedir = create_uniq_subdir(i->morguedir); } for( j = 0 ; j < i->files.count ; j ++ ) { char *fullfilename; if( !i->delete[j] ) continue; fullfilename = calc_dirconcat(i->directory, i->files.values[j]); if( fullfilename == NULL ) { result = RET_ERROR_OOM; continue; } if( morguedir != NULL && !i->processed[j] ) { char *newname = calc_dirconcat(morguedir, i->files.values[j]); if( newname != NULL && rename(fullfilename, newname) == 0 ) { free(newname); free(fullfilename); continue; } else if( newname == NULL ) { result = RET_ERROR_OOM; } else { int e = errno; fprintf(stderr, "Error %d moving '%s' to '%s': %s\n", e, i->files.values[j], morguedir, strerror(e)); RET_UPDATE(result, RET_ERRNO(e)); /* no continue, instead * delete the file as normal: */ } } if( verbose >= 3 ) printf("deleting '%s'...\n", fullfilename); deletefile(fullfilename); free(fullfilename); } if( morguedir != NULL ) { /* in the case it is empty, remove again */ (void)rmdir(morguedir); free(morguedir); } incoming_free(i); return result; }