/* This file is part of "reprepro" * Copyright (C) 2003,2004,2005,2006,2007,2008 Bernhard R. Link * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License version 2 as * published by the Free Software Foundation. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License * along with this program; if not, write to the Free Software * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02111-1301 USA */ #include #include #include #include #include #include #include #include #include #include #include #include "error.h" #include "strlist.h" #include "filecntl.h" #include "names.h" #include "checksums.h" #include "dirs.h" #include "names.h" #include "files.h" #include "ignore.h" #include "filelist.h" #include "debfile.h" #include "pool.h" #include "database_p.h" static retvalue files_get_checksums(struct database *database, const char *filekey, /*@out@*/struct checksums **checksums_p) { const char *md5sum = NULL; const char *checksums; size_t checksumslen; retvalue r; r = table_gettemprecord(database->checksums, filekey, &checksums, &checksumslen); if( !RET_IS_OK(r) ) return r; return checksums_setall(checksums_p, checksums, checksumslen, md5sum); } retvalue files_add_checksums(struct database *database, const char *filekey, const struct checksums *checksums) { retvalue r; const char *combined; size_t combinedlen; assert( database->checksums != NULL ); r = checksums_getcombined(checksums, &combined, &combinedlen); if( !RET_IS_OK(r) ) return r; r = table_adduniqsizedrecord(database->checksums, filekey, combined, combinedlen + 1, true, false); if( !RET_IS_OK(r) ) return r; return pool_markadded(filekey); } static retvalue files_replace_checksums(struct database *database, const char *filekey, const struct checksums *checksums) { retvalue r; const char *combined; size_t combinedlen; assert( database->checksums != NULL ); r = checksums_getcombined(checksums, &combined, &combinedlen); if( !RET_IS_OK(r) ) return r; return table_adduniqsizedrecord(database->checksums, filekey, combined, combinedlen + 1, true, false); } /* remove file's md5sum from database */ retvalue files_removesilent(struct database *database, const char *filekey) { retvalue r; if( database->contents != NULL ) (void)table_deleterecord(database->contents, filekey, true); r = table_deleterecord(database->checksums, filekey, true); if( r == RET_NOTHING ) { fprintf(stderr, "Unable to forget unknown filekey '%s'.\n", filekey); return RET_ERROR_MISSING; } return r; } retvalue files_remove(struct database *database, const char *filekey) { retvalue r; r = files_removesilent(database, filekey); if( RET_IS_OK(r) ) return pool_markdeleted(filekey); return r; } /* hardlink file with known checksums and add it to database */ retvalue files_hardlinkandadd(struct database *database, const char *tempfile, const char *filekey, const struct checksums *checksums) { retvalue r; /* an additional check to make sure nothing tricks us into * overwriting it by another file */ r = files_canadd(database, filekey, checksums); if( !RET_IS_OK(r) ) return r; r = checksums_hardlink(global.outdir, filekey, tempfile, checksums); if( RET_WAS_ERROR(r) ) return r; return files_add_checksums(database, filekey, checksums); } /* check if file is already there (RET_NOTHING) or could be added (RET_OK) * or RET_ERROR_WRONG_MD5SUM if filekey is already there with different md5sum */ retvalue files_canadd(struct database *database, const char *filekey, const struct checksums *checksums) { retvalue r; struct checksums *indatabase; bool improves; r = files_get_checksums(database, filekey, &indatabase); if( r == RET_NOTHING ) return RET_OK; if( RET_WAS_ERROR(r) ) return r; if( !checksums_check(indatabase, checksums, &improves) ) { fprintf(stderr, "File \"%s\" is already registered with different checksums!\n", filekey); checksums_printdifferences(stderr, indatabase, checksums); checksums_free(indatabase); return RET_ERROR_WRONG_MD5; } // TODO: sometimes the caller might want to have additional // checksums from the database already, think about ways to // make them available... checksums_free(indatabase); return RET_NOTHING; } /* check for file in the database and if not found there, if it can be detected */ retvalue files_expect(struct database *database, const char *filekey, const struct checksums *checksums, bool warnifadded) { retvalue r; char *filename; struct checksums *improvedchecksums = NULL; r = files_canadd(database, filekey, checksums); if( r == RET_NOTHING ) return RET_OK; if( RET_WAS_ERROR(r) ) return r; /* ready to add means missing, so have to look for the file itself: */ filename = files_calcfullfilename(filekey); if( filename == NULL ) return RET_ERROR_OOM; /* first check if a possible manually put (or left over from previous * downloads attepts) file is there and is correct */ r = checksums_test(filename, checksums, &improvedchecksums); if( r == RET_ERROR_WRONG_MD5) { fprintf(stderr, "Deleting unexpected file '%s'!\n" "(not in database and wrong in pool)\n ", filename); if( unlink(filename) == 0 ) r = RET_NOTHING; else { int e = errno; fprintf(stderr, "Error %d deleting '%s': %s!\n", e, filename, strerror(e)); } } free(filename); if( !RET_IS_OK(r) ) return r; if( warnifadded ) fprintf(stderr, "Warning: readded existing file '%s' mysteriously missing from the checksum database.\n", filekey); // TODO: some callers might want the updated checksum when // improves is true, how to get them there? /* add found file to database */ if( improvedchecksums != NULL ) { r = files_add_checksums(database, filekey, improvedchecksums); checksums_free(improvedchecksums); } else r = files_add_checksums(database, filekey, checksums); assert( r != RET_NOTHING ); return r; } /* check for several files in the database and in the pool if missing */ retvalue files_expectfiles(struct database *database, const struct strlist *filekeys, struct checksums *checksumsarray[]) { int i; retvalue r; for( i = 0 ; i < filekeys->count ; i++ ) { const char *filekey = filekeys->values[i]; const struct checksums *checksums = checksumsarray[i]; r = files_expect(database, filekey, checksums, verbose >= 0); if( RET_WAS_ERROR(r) ) return r; if( r == RET_NOTHING ) { /* File missing */ fprintf(stderr, "Missing file %s\n", filekey); return RET_ERROR_MISSING; } } return RET_OK; } /* print missing files */ retvalue files_printmissing(struct database *database, const struct strlist *filekeys, const struct checksumsarray *origfiles) { int i; retvalue ret,r; ret = RET_NOTHING; assert( filekeys->count == origfiles->names.count ); for( i = 0 ; i < filekeys->count ; i++ ) { const char *filekey = filekeys->values[i]; const char *origfile = origfiles->names.values[i]; const struct checksums *checksums = origfiles->checksums[i]; r = files_expect(database, filekey, checksums, false); if( RET_WAS_ERROR(r) ) { return r; } if( r == RET_NOTHING ) { /* File missing */ (void)fputs(origfile, stdout); (void)putchar(' '); (void)fputs(global.outdir, stdout); (void)putchar('/'); (void)fputs(filekey, stdout); (void)putchar('\n'); RET_UPDATE(ret,RET_OK); } else RET_UPDATE(ret,r); } return ret; } /* dump out all information */ retvalue files_printmd5sums(struct database *database) { retvalue result, r; struct cursor *cursor; const char *filekey, *checksum; r = table_newglobalcursor(database->checksums, &cursor); if( !RET_IS_OK(r) ) return r; result = RET_NOTHING; while( cursor_nexttemp(database->checksums, cursor, &filekey, &checksum) ) { result = RET_OK; (void)fputs(filekey, stdout); (void)putchar(' '); while( *checksum == ':' ) { while( *checksum != ' ' && *checksum != '\0' ) checksum++; if( *checksum == ' ' ) checksum++; } (void)fputs(checksum, stdout); (void)putchar('\n'); } r = cursor_close(database->checksums, cursor); RET_ENDUPDATE(result, r); return result; } retvalue files_printchecksums(struct database *database) { retvalue result, r; struct cursor *cursor; const char *filekey, *checksum; r = table_newglobalcursor(database->checksums, &cursor); if( !RET_IS_OK(r) ) return r; result = RET_NOTHING; while( cursor_nexttemp(database->checksums, cursor, &filekey, &checksum) ) { result = RET_OK; (void)fputs(filekey, stdout); (void)putchar(' '); (void)fputs(checksum, stdout); (void)putchar('\n'); if( interrupted() ) { result = RET_ERROR_INTERRUPTED; break; } } r = cursor_close(database->checksums, cursor); RET_ENDUPDATE(result, r); return result; } /* callback for each registered file */ retvalue files_foreach(struct database *database,per_file_action action,void *privdata) { retvalue result, r; struct cursor *cursor; const char *filekey, *checksum; r = table_newglobalcursor(database->checksums, &cursor); if( !RET_IS_OK(r) ) return r; result = RET_NOTHING; while( cursor_nexttemp(database->checksums, cursor, &filekey, &checksum) ) { if( interrupted() ) { RET_UPDATE(result, RET_ERROR_INTERRUPTED); break; } r = action(privdata, filekey); RET_UPDATE(result, r); } r = cursor_close(database->checksums, cursor); RET_ENDUPDATE(result, r); return result; } static retvalue checkpoolfile(const char *fullfilename, const struct checksums *expected, bool *improveable) { struct checksums *actual; retvalue r; bool improves; r = checksums_read(fullfilename, &actual); if( RET_IS_OK(r) ) { if( !checksums_check(expected, actual, &improves) ) { fprintf(stderr, "WRONG CHECKSUMS of '%s':\n", fullfilename); checksums_printdifferences(stderr, expected, actual); r = RET_ERROR_WRONG_MD5; } else if( improves ) *improveable = true; checksums_free(actual); } return r; } retvalue files_checkpool(struct database *database, bool fast) { retvalue result, r; struct cursor *cursor; const char *filekey, *combined; size_t combinedlen; struct checksums *expected; char *fullfilename; bool improveable = false; result = RET_NOTHING; r = table_newglobalcursor(database->checksums, &cursor); if( !RET_IS_OK(r) ) return r; while( cursor_nexttempdata(database->checksums, cursor, &filekey, &combined, &combinedlen) ) { r = checksums_setall(&expected, combined, combinedlen, NULL); if( RET_WAS_ERROR(r) ) { RET_UPDATE(result, r); continue; } fullfilename = files_calcfullfilename(filekey); if( fullfilename == NULL ) { result = RET_ERROR_OOM; checksums_free(expected); break; } if( fast ) r = checksums_cheaptest(fullfilename, expected, true); else r = checkpoolfile(fullfilename, expected, &improveable); if( r == RET_NOTHING ) { fprintf(stderr, "Missing file '%s'!\n", fullfilename); r = RET_ERROR_MISSING; } free(fullfilename); checksums_free(expected); RET_UPDATE(result, r); } r = cursor_close(database->checksums, cursor); RET_ENDUPDATE(result, r); if( improveable && verbose >= 0 ) printf( "There were files with only some of the checksums this version of reprepro\n" "can compute recorded. To add those run reprepro collectnewchecksums.\n"); return result; } retvalue files_collectnewchecksums(struct database *database) { retvalue result, r; struct cursor *cursor; const char *filekey, *all; size_t alllen; struct checksums *expected; char *fullfilename; result = RET_NOTHING; r = table_newglobalcursor(database->checksums, &cursor); if( !RET_IS_OK(r) ) return r; while( cursor_nexttempdata(database->checksums, cursor, &filekey, &all, &alllen) ) { r = checksums_setall(&expected, all, alllen, NULL); if( !RET_IS_OK(r) ) { RET_UPDATE(result, r); continue; } if( checksums_iscomplete(expected) ) { checksums_free(expected); continue; } fullfilename = files_calcfullfilename(filekey); if( fullfilename == NULL ) { result = RET_ERROR_OOM; checksums_free(expected); break; } r = checksums_complete(&expected, fullfilename); if( r == RET_NOTHING ) { fprintf(stderr, "Missing file '%s'!\n", fullfilename); r = RET_ERROR_MISSING; } if( r == RET_ERROR_WRONG_MD5 ) { fprintf(stderr, "ERROR: Cannot collect missing checksums for '%s'\n" "as the file in the pool does not match the already recorded checksums\n", filekey); } free(fullfilename); if( RET_IS_OK(r) ) r = files_replace_checksums(database, filekey, expected); checksums_free(expected); RET_UPDATE(result, r); } r = cursor_close(database->checksums, cursor); RET_ENDUPDATE(result, r); return result; } retvalue files_detect(struct database *database, const char *filekey) { struct checksums *checksums; char *fullfilename; retvalue r; fullfilename = files_calcfullfilename(filekey); if( fullfilename == NULL ) return RET_ERROR_OOM; r = checksums_read(fullfilename, &checksums); if( r == RET_NOTHING ) { fprintf(stderr, "Error opening '%s'!\n", fullfilename); r = RET_ERROR_MISSING; } if( RET_WAS_ERROR(r) ) { free(fullfilename); return r; } free(fullfilename); r = files_add_checksums(database, filekey, checksums); checksums_free(checksums); return r; } struct rfd { bool reread; /*@temp@*/struct database *database; }; static retvalue regenerate_filelist(void *data, const char *filekey) { struct database *database = ((struct rfd*)data)->database; bool reread = ((struct rfd*)data)->reread; size_t l = strlen(filekey); char *debfilename; char *filelist; size_t fls; retvalue r; if( l <= 4 || memcmp(filekey+l-4, ".deb", 4) != 0 ) return RET_NOTHING; if( !reread && !table_recordexists(database->contents, filekey) ) return RET_NOTHING; debfilename = files_calcfullfilename(filekey); if( debfilename == NULL ) return RET_ERROR_OOM; r = getfilelist(&filelist, &fls, debfilename); free(debfilename); if( RET_IS_OK(r) ) { if( verbose > 0 ) (void)puts(filekey); if( verbose > 6 ) { const char *p = filelist; while( *p != '\0' ) { (void)putchar(' '); (void)puts(p); p += strlen(p)+1; } } r = table_adduniqsizedrecord( database->contents, filekey, filelist, fls, true, true); free(filelist); } return r; } retvalue files_regenerate_filelist(struct database *database, bool reread) { struct rfd d; d.database = database; d.reread = reread; return files_foreach(database, regenerate_filelist, &d); } /* Include a yet unknown file into the pool */ retvalue files_preinclude(struct database *database, const char *sourcefilename, const char *filekey, struct checksums **checksums_p) { retvalue r; struct checksums *checksums, *realchecksums; bool improves; char *fullfilename; r = files_get_checksums(database, filekey, &checksums); if( RET_WAS_ERROR(r) ) return r; if( RET_IS_OK(r) ) { r = checksums_read(sourcefilename, &realchecksums); if( r == RET_NOTHING ) r = RET_ERROR_MISSING; if( RET_WAS_ERROR(r) ) { checksums_free(checksums); return r; } if( !checksums_check(checksums, realchecksums, &improves) ) { fprintf(stderr, "ERROR: '%s' cannot be included as '%s'.\n" "Already existing files can only be included again, if they are the same, but:\n", sourcefilename, filekey); checksums_printdifferences(stderr, checksums, realchecksums); checksums_free(checksums); checksums_free(realchecksums); return RET_ERROR_WRONG_MD5; } if( improves ) { r = checksums_combine(&checksums, realchecksums, NULL); if( RET_WAS_ERROR(r) ) { checksums_free(realchecksums); checksums_free(checksums); return r; } r = files_replace_checksums(database, filekey, checksums); if( RET_WAS_ERROR(r) ) { checksums_free(realchecksums); checksums_free(checksums); return r; } } checksums_free(realchecksums); // args, this breaks retvalue semantics! if( checksums_p != NULL ) *checksums_p = checksums; else checksums_free(checksums); return RET_NOTHING; } assert( sourcefilename != NULL ); fullfilename = files_calcfullfilename(filekey); if( fullfilename == NULL ) return RET_ERROR_OOM; (void)dirs_make_parent(fullfilename); r = checksums_copyfile(fullfilename, sourcefilename, true, &checksums); if( r == RET_ERROR_EXIST ) { // TODO: deal with already existing files! fprintf(stderr, "File '%s' does already exist!\n", fullfilename); } if( r == RET_NOTHING ) { fprintf(stderr, "Could not open '%s'!\n", sourcefilename); r = RET_ERROR_MISSING; } if( RET_WAS_ERROR(r) ) { free(fullfilename); return r; } free(fullfilename); r = files_add_checksums(database, filekey, checksums); if( RET_WAS_ERROR(r) ) { checksums_free(checksums); return r; } if( checksums_p != NULL ) *checksums_p = checksums; else checksums_free(checksums); return RET_OK; } static retvalue checkimproveorinclude(const char *sourcedir, const char *basefilename, const char *filekey, struct checksums **checksums_p, bool *improving) { retvalue r; struct checksums *checksums = NULL; bool improves, copied = false; char *fullfilename = files_calcfullfilename(filekey); if( fullfilename == NULL ) return RET_ERROR_OOM; if( checksums_iscomplete(*checksums_p) ) { r = checksums_cheaptest(fullfilename, *checksums_p, true); if( r != RET_NOTHING ) { free(fullfilename); return r; } } else { r = checksums_read(fullfilename, &checksums); if( RET_WAS_ERROR(r) ) { free(fullfilename); return r; } } if( r == RET_NOTHING ) { char *sourcefilename = calc_dirconcat(sourcedir, basefilename); if( sourcefilename == NULL ) { free(fullfilename); return RET_ERROR_OOM; } fprintf(stderr, "WARNING: file %s was lost!\n" "(i.e. found in the database, but not in the pool)\n" "trying to compensate...\n", filekey); (void)dirs_make_parent(fullfilename); r = checksums_copyfile(fullfilename, sourcefilename, false, &checksums); if( r == RET_ERROR_EXIST ) { fprintf(stderr, "File '%s' seems to be missing and existing at the same time!\n" "To confused to continue...\n", fullfilename); } if( r == RET_NOTHING ) { fprintf(stderr, "Could not open '%s'!\n", sourcefilename); r = RET_ERROR_MISSING; } free(sourcefilename); if( RET_WAS_ERROR(r) ) { free(fullfilename); return r; } copied = true; } assert( checksums != NULL ); if( !checksums_check(*checksums_p, checksums, &improves) ) { if( copied ) { deletefile(fullfilename); fprintf(stderr, "ERROR: Unexpected content of file '%s/%s'!\n", sourcedir, basefilename); } else // TODO: if the database only listed some of the currently supported checksums, // and the caller of checkincludefile supplied some (which none yet does), but // not all (which needs at least three checksums, i.e. not applicaple before // sha256 get added), then this might also be called if the file in the pool // just has the same checksums as previously recorded (e.g. a md5sum collision) // but the new file was listed with another secondary hash than the original. // In that situation it might be a bit misleading... fprintf(stderr, "ERROR: file %s is damaged!\n" "(i.e. found in the database, but with different checksums in the pool)\n", filekey); checksums_printdifferences(stderr, *checksums_p, checksums); r = RET_ERROR_WRONG_MD5; } if( improves ) { r = checksums_combine(checksums_p, checksums, NULL); if( RET_IS_OK(r) ) *improving = true; } checksums_free(checksums); free(fullfilename); return r; } retvalue files_checkincludefile(struct database *database, const char *sourcedir, const char *basefilename, const char *filekey, struct checksums **checksums_p) { char *sourcefilename, *fullfilename; struct checksums *checksums; retvalue r; bool improves; assert( *checksums_p != NULL ); r = files_get_checksums(database, filekey, &checksums); if( RET_WAS_ERROR(r) ) return r; if( RET_IS_OK(r) ) { /* there are three sources now: * - the checksums from the database (may have some we * do not even know about, and may miss some we can * generate) * - the checksums provided (typically only md5sum, * as this comes from a .changes or .dsc) * - the checksums of the file * * to make things more complicated, the file should only * be read if needed, as this needs time. * And it can happen the file got lost in the pool, then * this is the best place to replace it. */ if( !checksums_check(checksums, *checksums_p, &improves) ) { fprintf(stderr, "ERROR: '%s/%s' cannot be included as '%s'.\n" "Already existing files can only be included again, if they are the same, but:\n", sourcedir, basefilename, filekey); checksums_printdifferences(stderr, checksums, *checksums_p); checksums_free(checksums); return RET_ERROR_WRONG_MD5; } r = RET_NOTHING; if( improves ) r = checksums_combine(&checksums, *checksums_p, NULL); if( !RET_WAS_ERROR(r) ) r = checkimproveorinclude(sourcedir, basefilename, filekey, &checksums, &improves); if( !RET_WAS_ERROR(r) && improves ) r = files_replace_checksums(database, filekey, checksums); if( RET_IS_OK(r) ) r = RET_NOTHING; /* return the combined checksum */ checksums_free(*checksums_p); *checksums_p = checksums; return r; } assert( sourcedir != NULL ); sourcefilename = calc_dirconcat(sourcedir, basefilename); if( sourcefilename == NULL ) return RET_ERROR_OOM; fullfilename = files_calcfullfilename(filekey); if( fullfilename == NULL ) { free(sourcefilename); return RET_ERROR_OOM; } (void)dirs_make_parent(fullfilename); r = checksums_copyfile(fullfilename, sourcefilename, true, &checksums); if( r == RET_NOTHING ) { fprintf(stderr, "Could not open '%s'!\n", sourcefilename); r = RET_ERROR_MISSING; } if( RET_WAS_ERROR(r) ) { free(fullfilename); free(sourcefilename); return r; } if( !checksums_check(*checksums_p, checksums, &improves) ) { deletefile(fullfilename); fprintf(stderr, "ERROR: Unexpected content of file '%s'!\n", sourcefilename); checksums_printdifferences(stderr, *checksums_p, checksums); r = RET_ERROR_WRONG_MD5; } free(sourcefilename); free(fullfilename); if( RET_WAS_ERROR(r) ) { return r; } if( improves ) { r = checksums_combine(checksums_p, checksums, NULL); checksums_free(checksums); if( RET_WAS_ERROR(r) ) return r; } else checksums_free(checksums); return files_add_checksums(database, filekey, *checksums_p); }