From 217f4ef4605f52e62791721e9ee567ad643530db Mon Sep 17 00:00:00 2001 From: espie Date: Sat, 16 Apr 2022 09:32:40 +0000 Subject: [PATCH] refactor the code so that caching is setup in the repository that's linked to quirks during do_quirks. then add a pre-cache that does run locate once on the full list of things we want to update/install (assuming "regular" pkgsets, that is, where the names are the ones we want). Then the actual fetching of caching info will look in that pre-cache before resorting to locate itself (or going straight to the wire). The code isn't active yet, the XXX comment in OpenBSD/PackageRepository/Installed.pm says it all. --- usr.sbin/pkg_add/Makefile | 3 +- usr.sbin/pkg_add/OpenBSD/PackageLocation.pm | 40 +----- .../OpenBSD/PackageRepository/Cache.pm | 114 ++++++++++++++++++ .../OpenBSD/PackageRepository/Installed.pm | 24 +++- usr.sbin/pkg_add/OpenBSD/PkgAdd.pm | 6 +- 5 files changed, 145 insertions(+), 42 deletions(-) create mode 100644 usr.sbin/pkg_add/OpenBSD/PackageRepository/Cache.pm diff --git a/usr.sbin/pkg_add/Makefile b/usr.sbin/pkg_add/Makefile index 546cd0f0e0f..d3d6aec2c60 100644 --- a/usr.sbin/pkg_add/Makefile +++ b/usr.sbin/pkg_add/Makefile @@ -1,4 +1,4 @@ -# $OpenBSD: Makefile,v 1.93 2022/01/21 17:41:41 espie Exp $ +# $OpenBSD: Makefile,v 1.94 2022/04/16 09:32:40 espie Exp $ .include @@ -37,6 +37,7 @@ PACKAGES= \ OpenBSD/PackageLocator.pm \ OpenBSD/PackageName.pm \ OpenBSD/PackageRepository.pm \ + OpenBSD/PackageRepository/Cache.pm \ OpenBSD/PackageRepository/HTTP.pm \ OpenBSD/PackageRepository/Installed.pm \ OpenBSD/PackageRepository/Persistent.pm \ diff --git a/usr.sbin/pkg_add/OpenBSD/PackageLocation.pm b/usr.sbin/pkg_add/OpenBSD/PackageLocation.pm index 1a7a2d902d8..52fb2cdc6bb 100644 --- a/usr.sbin/pkg_add/OpenBSD/PackageLocation.pm +++ b/usr.sbin/pkg_add/OpenBSD/PackageLocation.pm @@ -1,5 +1,5 @@ # ex:ts=8 sw=4: -# $OpenBSD: PackageLocation.pm,v 1.55 2022/04/15 10:54:00 espie Exp $ +# $OpenBSD: PackageLocation.pm,v 1.56 2022/04/16 09:32:40 espie Exp $ # # Copyright (c) 2003-2007 Marc Espie # @@ -60,10 +60,8 @@ OpenBSD::Auto::cache(update_info, if ($self->name =~ /^quirks\-/) { return $self->plist; } - if ($self->{repository}{is_cached}) { - my $info = $self->get_update_info_fromdb; - return $info if defined $info; - } + my $info = $self->{repository}->get_cached_info($self->name); + return $info if defined $info; return $self->plist(\&OpenBSD::PackingList::UpdateInfoOnly, sub { return 0 if $_[0] =~ m/^\@option\s+always-update\b/m; @@ -72,38 +70,6 @@ OpenBSD::Auto::cache(update_info, }); }); -sub set_db_location -{ - my ($class, $location) = @_; - - # XXX UNCOMMENT THE NEXT LINE TO TURN ON CACHING -# $location->{repository}{is_cached} = 1; -} - -sub get_update_info_fromdb -{ - my $self = shift; - open my $fh, "-|", OpenBSD::Paths->locate, - '-d', OpenBSD::Paths->updateinfodb, $self->name.":*"; - my $content = ''; - while (<$fh>) { - if (m/\@option\s+always-update/) { - return undef; - } - if (m/^.*?\:(.*)/) { - $content .= $1."\n"; - } else { - return undef; - } - } - close ($fh); - if ($content eq '') { - return undef; - } - open my $fh2, "<", \$content; - return OpenBSD::PackingList->read($fh2); -} - # make sure self is opened and move to the right location if need be. sub _opened { diff --git a/usr.sbin/pkg_add/OpenBSD/PackageRepository/Cache.pm b/usr.sbin/pkg_add/OpenBSD/PackageRepository/Cache.pm new file mode 100644 index 00000000000..f01d87f2c21 --- /dev/null +++ b/usr.sbin/pkg_add/OpenBSD/PackageRepository/Cache.pm @@ -0,0 +1,114 @@ +# ex:ts=8 sw=4: +# $OpenBSD: Cache.pm,v 1.1 2022/04/16 09:32:40 espie Exp $ +# +# Copyright (c) 2022 Marc Espie +# +# Permission to use, copy, modify, and distribute this software for any +# purpose with or without fee is hereby granted, provided that the above +# copyright notice and this permission notice appear in all copies. +# +# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES +# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF +# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR +# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES +# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN +# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF +# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + +use strict; +use warnings; + +# supplementary glue to add support for reading the update.db locate(1) +# database in quirks +package OpenBSD::PackageRepository::Cache; + +sub new +{ + my ($class, $state, $setlist) = @_; + + return undef unless -f OpenBSD::Paths->updateinfodb; + + my $o = bless { + raw_data => {}, + state => $state, + uncached => {}}, $class; + + $o->prime_update_info_cache($state, $setlist); + return $o; + +} +sub prime_update_info_cache +{ + my ($self, $state, $setlist) = @_; + + my $progress = $state->progress; + my $stems = {}; + # figure out a list of names to precache + + # okay, so basically instead of hitting locate once for each + # package on the distant repository, we precache all the stems + # we are asking to update/install + # this is based on the assumption that most names are "regular" + # and we won't cache too little or too much + for my $set (@{$setlist}) { + for my $h ($set->older, $set->hints) { + next if $h->{update_found}; + my $name = $h->pkgname; + $stems->{OpenBSD::PackageName::splitstem($h->{pkgname})} = 1; + } + } + my @list = keys %$stems; + $progress->set_header( + $state->f("Precaching update information for #1 names...", + scalar(@list))); + open my $fh, "-|", OpenBSD::Paths->locate, + '-d', OpenBSD::Paths->updateinfodb, map { "$_-[0-9]*"} @list; + while (<$fh>) { + $progress->working(100); + if (m/^(.*?)\:(.*)/) { + my ($pkgname, $value) = ($1, $2); + $self->{raw_data}{$pkgname} //= ''; + $self->{raw_data}{$pkgname} .= "$value\n"; + if ($value =~ m/\@option\s+always-update/) { + $self->{uncached}{$pkgname} = 1; + } + } + } + close($fh); +} + +sub get_cached_info +{ + my ($self, $name) = @_; + + if ($self->{uncached}{$name}) { + return undef; + } + my $content; + if (exists $self->{raw_data}{$name}) { + $content = $self->{raw_data}{$name}; + } else { + $self->{state}->errsay("Calling locate on #1", $name); + $content = ''; + open my $fh, "-|", OpenBSD::Paths->locate, + '-d', OpenBSD::Paths->updateinfodb, $name.":*"; + while (<$fh>) { + if (m/\@option\s+always-update/) { + return undef; + } + if (m/^.*?\:(.*)/) { + $content .= $1."\n"; + } else { + return undef; + } + } + close ($fh); + } + if ($content eq '') { + return undef; + } + open my $fh2, "<", \$content; + return OpenBSD::PackingList->read($fh2); +} + +1; diff --git a/usr.sbin/pkg_add/OpenBSD/PackageRepository/Installed.pm b/usr.sbin/pkg_add/OpenBSD/PackageRepository/Installed.pm index 80d40a25563..008c1856585 100644 --- a/usr.sbin/pkg_add/OpenBSD/PackageRepository/Installed.pm +++ b/usr.sbin/pkg_add/OpenBSD/PackageRepository/Installed.pm @@ -1,5 +1,5 @@ # ex:ts=8 sw=4: -# $OpenBSD: Installed.pm,v 1.38 2017/03/11 11:25:01 espie Exp $ +# $OpenBSD: Installed.pm,v 1.39 2022/04/16 09:32:40 espie Exp $ # # Copyright (c) 2007-2014 Marc Espie # @@ -50,6 +50,28 @@ sub expand_locations } } +sub get_cached_info +{ + my ($repository, $name) = @_; + if (defined $repository->{info_cache}) { + return $repository->{info_cache}->get_cached_info($name); + } else { + return undef; + } +} + +sub setup_cache +{ + my ($repository, $setlist) = @_; + # XXX remove the next line to actually get the cache + return; + require OpenBSD::PackageRepository::Cache; + + $repository->{info_cache} = + OpenBSD::PackageRepository::Cache->new( + $repository->{state}, $setlist); +} + sub parse_url { my ($class, $r, $state) = @_; diff --git a/usr.sbin/pkg_add/OpenBSD/PkgAdd.pm b/usr.sbin/pkg_add/OpenBSD/PkgAdd.pm index 9a14ae91a1b..8d43eb9f7c9 100644 --- a/usr.sbin/pkg_add/OpenBSD/PkgAdd.pm +++ b/usr.sbin/pkg_add/OpenBSD/PkgAdd.pm @@ -1,7 +1,7 @@ #! /usr/bin/perl # ex:ts=8 sw=4: -# $OpenBSD: PkgAdd.pm,v 1.127 2022/04/15 08:03:33 espie Exp $ +# $OpenBSD: PkgAdd.pm,v 1.128 2022/04/16 09:32:40 espie Exp $ # # Copyright (c) 2003-2014 Marc Espie # @@ -363,7 +363,7 @@ sub find_kept_handle if ($set->{quirks}) { # The installed package has inst: for a location, we want # the newer one (which is identical) - OpenBSD::PackageLocation->set_db_location($n->location); + $n->location->{repository}->setup_cache($state->{setlist}); } $set->move_kept($o); $o->{tweaked} = @@ -861,7 +861,7 @@ sub really_add delete $handle->{partial}; OpenBSD::PkgCfl::register($handle, $state); if ($set->{quirks}) { - OpenBSD::PackageLocation->set_db_location($handle->location); + $handle->location->{repository}->setup_cache($state->{setlist}); } } delete $state->{partial}; -- 2.20.1