lazyload experiment: separate libraries

FossilOrigin-Name: 8660559c9b7b280bd5113dfcb54bb4a0a2ef0bda65036c0fbe6c1a4dfc9ba8a1

1 files changed, 879 insertions, 0 deletions

diff --git a/lib/cpt-pkg b/lib/cpt-pkg
new file mode 100644
index 0000000..bfaa428
--- /dev/null
+++ b/lib/cpt-pkg
@@ -0,0 +1,879 @@
+# Package functions
+# shellcheck source=lib-cpt
+
+run_hook() {
+    # Store the CPT_HOOK variable so that we can revert it if it is changed.
+    oldCPT_HOOK=$CPT_HOOK
+
+    # If a fourth parameter 'root' is specified, source the hook from a
+    # predefined location to avoid privilige escalation through user scripts.
+    [ "$4" ] && CPT_HOOK=$CPT_ROOT/etc/cpt-hook
+
+    [ -f "$CPT_HOOK" ] || { CPT_HOOK=$oldCPT_HOOK; return 0 ;}
+
+    [ "$2" ] && log "$2" "Running $1 hook"
+
+    TYPE=${1:-null} PKG=${2:-null} DEST=${3:-null} . "$CPT_HOOK"
+    CPT_HOOK=$oldCPT_HOOK
+}
+
+pkg_lint() {
+    # Check that each mandatory file in the package entry exists.
+    log "$1" "Checking repository files"
+
+    repo_dir=$(pkg_find "$1")
+
+    cd "$repo_dir" || die "'$repo_dir' not accessible"
+    [ -f sources ] || warn "$1" "Sources file not found"
+    [ -x build ]   || die "$1" "Build file not found or not executable"
+    [ -s version ] || die "$1" "Version file not found or empty"
+
+    read -r _ release 2>/dev/null < version || die "Version file not found"
+    [ "$release" ] || die "Release field not found in version file"
+
+    [ "$2" ] || [ -f checksums ] || die "$pkg" "Checksums are missing"
+}
+
+pkg_cache() {
+    read -r version release 2>/dev/null < "$(pkg_find "$1")/version"
+
+    # Initially assume that the package tarball is built with the CPT_COMPRESS
+    # value.
+    if [ -f "$bin_dir/$1#$version-$release.tar.$CPT_COMPRESS" ]; then
+        tar_file="$bin_dir/$1#$version-$release.tar.$CPT_COMPRESS"
+    else
+        set +f; set -f -- "$bin_dir/$1#$version-$release.tar."*
+        tar_file=$1
+    fi
+
+    [ -f "$tar_file" ]
+}
+
+pkg_strip() {
+    # Strip package binaries and libraries. This saves space on the
+    # system as well as on the tarballs we ship for installation.
+
+    # Package has stripping disabled, stop here.
+    [ -f "$mak_dir/$pkg/nostrip" ] && return
+
+    log "$1" "Stripping binaries and libraries"
+
+    find "$pkg_dir/$1" -type f | while read -r file; do
+        case $(od -A o -t c -N 18 "$file") in
+            # REL (object files (.o), static libraries (.a)).
+            *177*E*L*F*0000020\ 001\ *|*\!*\<*a*r*c*h*\>*)
+                strip -g -R .comment -R .note "$file"
+            ;;
+
+            # EXEC (static binaries).
+            # DYN (shared libraries, dynamic binaries).
+            # Shared libraries keep global symbols in a separate ELF section
+            # called '.dynsym'. '--strip-all/-s' does not touch the dynamic
+            # symbol entries which makes this safe to do.
+            *177*E*L*F*0000020\ 00[23]\ *)
+                strip -s -R .comment -R .note "$file"
+            ;;
+        esac
+    done 2>/dev/null ||:
+}
+
+pkg_manifest() (
+    # Generate the package's manifest file. This is a list of each file
+    # and directory inside the package. The file is used when uninstalling
+    # packages, checking for package conflicts and for general debugging.
+    log "$1" "Generating manifest"
+
+    # This function runs as a sub-shell to avoid having to 'cd' back to the
+    # prior directory before being able to continue.
+    cd "${2:-$pkg_dir}/$1"
+
+    # find: Print all files and directories and append '/' to directories.
+    # sort: Sort the output in *reverse*. Directories appear *after* their
+    #       contents.
+    # sed:  Remove the first character in each line (./dir -> /dir) and
+    #       remove all lines which only contain '.'.
+    find . -type d -exec printf '%s/\n' {} + -o -print |
+        sort -r | sed '/^\.\/$/d;ss.ss' > "${2:-$pkg_dir}/$1/$pkg_db/$1/manifest"
+)
+
+pkg_etcsums() (
+    # This function runs as a sub-shell to avoid having to 'cd' back to the
+    # prior directory before being able to continue.
+    cd "$pkg_dir/$1/etc" 2>/dev/null || return 0; cd ..
+
+    # Generate checksums for each configuration file in the package's
+    # /etc/ directory for use in "smart" handling of these files.
+    log "$1" "Generating etcsums"
+
+
+    find etc -type f | while read -r file; do
+        sh256 "$file"
+    done > "$pkg_dir/$1/$pkg_db/$1/etcsums"
+)
+
+pkg_tar() {
+    # Create a tarball from the built package's files.
+    # This tarball also contains the package's database entry.
+    log "$1" "Creating tarball"
+
+    # Read the version information to name the package.
+    read -r version release < "$(pkg_find "$1")/version"
+
+    # Create a tarball from the contents of the built package.
+    "$tar" cf - -C "$pkg_dir/$1" . |
+        case $CPT_COMPRESS in
+            bz2) bzip2 -z ;;
+            xz)  xz -zT 0 ;;
+            gz)  gzip -6  ;;
+            zst) zstd -3  ;;
+            *)   gzip -6  ;;  # Fallback to gzip
+        esac \
+    > "$bin_dir/$1#$version-$release.tar.$CPT_COMPRESS"
+
+    log "$1" "Successfully created tarball"
+
+    run_hook post-package "$1" "$bin_dir/$1#$version-$release.tar.$CPT_COMPRESS"
+}
+
+pkg_build() {
+    # Build packages and turn them into packaged tarballs. This function
+    # also checks checksums, downloads sources and ensure all dependencies
+    # are installed.
+    pkg_build=1
+
+    log "Resolving dependencies"
+
+    for pkg do contains "$explicit" "$pkg" || {
+        pkg_depends "$pkg" explicit
+
+        # Mark packages passed on the command-line
+        # separately from those detected as dependencies.
+        explicit="$explicit $pkg "
+    } done
+
+    [ "$pkg_update" ] || explicit_build=$explicit
+
+    # If an explicit package is a dependency of another explicit
+    # package, remove it from the explicit list as it needs to be
+    # installed as a dependency.
+    # shellcheck disable=2086,2031
+    for pkg do
+        contains "$deps" "$pkg" && explicit=$(pop "$pkg" from $explicit)
+    done
+
+    # See [1] at top of script.
+    # shellcheck disable=2046,2086,2031
+    set -- $deps $explicit
+
+    log "Building: $*"
+
+    # Only ask for confirmation if more than one package needs to be built.
+    [ $# -gt 1 ] || [ "$pkg_update" ] && { prompt || exit 0 ;}
+
+    log "Checking for pre-built dependencies"
+
+    for pkg do pkg_lint "$pkg"; done
+
+    # Install any pre-built dependencies if they exist in the binary
+    # directory and are up to date.
+    for pkg do ! contains "$explicit_build" "$pkg" && pkg_cache "$pkg" && {
+        log "$pkg" "Found pre-built binary, installing"
+        (CPT_FORCE=1 cpt-install "$tar_file")
+
+        # Remove the now installed package from the build list.
+        # See [1] at top of script.
+        # shellcheck disable=2046,2086
+        set -- $(pop "$pkg" from "$@")
+    } done
+
+    for pkg do pkg_sources "$pkg"; done
+
+    pkg_verify "$@"
+
+    # Finally build and create tarballs for all passed packages and
+    # dependencies.
+    for pkg do
+        log "$pkg" "Building package ($((in = in + 1))/$#)"
+
+        pkg_extract "$pkg"
+        repo_dir=$(pkg_find "$pkg")
+
+        read -r build_version _ < "$repo_dir/version"
+
+        # Copy the build file to the build directory to users to modify it
+        # temporarily at runtime.
+        cp -f "$repo_dir/build" "$mak_dir/$pkg/.build.cpt"
+
+        # Install built packages to a directory under the package name
+        # to avoid collisions with other packages.
+        mkdir -p "$pkg_dir/$pkg/$pkg_db"
+
+        # Move to the build directory.
+        cd "$mak_dir/$pkg"
+
+        log "$pkg" "Starting build"
+
+        run_hook pre-build "$pkg" "$pkg_dir/$pkg"
+
+        # Notify the user if the build script is changed during the pre-build
+        # hook.
+        diff -q "$repo_dir/build" .build.cpt ||
+            log "$pkg" "Executing the modified build file"
+
+        # Call the build script, log the output to the terminal
+        # and to a file. There's no PIPEFAIL in POSIX shelll so
+        # we must resort to tricks like killing the script ourselves.
+        { ./.build.cpt "$pkg_dir/$pkg" "$build_version" "$sys_arch" 2>&1 || {
+            log "$pkg" "Build failed"
+            log "$pkg" "Log stored to $log_dir/$pkg-$time-$pid"
+            run_hook build-fail "$pkg" "$pkg_dir/$pkg"
+            pkg_clean
+            kill 0
+        } } | tee "$log_dir/$pkg-$time-$pid"
+
+        # Run the test script if it exists and the user wants to run tests. This
+        # is turned off by default.
+        [ -x "$repo_dir/test" ] && [ "$CPT_TEST" = 1 ] && {
+            run_hook pre-test "$pkg" "$pkg_dir/$pkg"
+            log "$pkg" "Running tests"
+            "$repo_dir/test" "$pkg_dir/$pkg" "$build_version" "$sys_arch" 2>&1 || {
+                log "$pkg" "Test failed"
+                log "$pkg" "Log stored to $log_dir/$pkg-$time-$pid"
+                run_hook test-fail "$pkg" "$pkg_dir/$pkg"
+                pkg_clean
+                kill 0
+        } } | tee -a "$log_dir/$pkg-$time-$pid"
+
+        # Delete the log file if the build succeeded to prevent
+        # the directory from filling very quickly with useless logs.
+        [ "$CPT_KEEPLOG" = 1 ] || rm -f "$log_dir/$pkg-$time-$pid"
+
+        # Copy the repository files to the package directory.
+        # This acts as the database entry.
+        cp -LRf "$repo_dir" "$pkg_dir/$pkg/$pkg_db/"
+
+        # Copy the modified build file to the package directory.
+        pkg_build="$pkg_dir/$pkg/$pkg_db/$pkg/build"
+        diff -U 3 "$pkg_build" .build.cpt > "$pkg_build.diff" &&
+            rm -f "$pkg_build.diff"
+
+        # We don't want the package manager to track 'dir' pages of the info
+        # directory. We don't want every single package to create their own dir
+        # files either.
+        rm -f "$pkg_dir/$pkg/usr/share/info/dir"
+
+        # We never ever want this. Let's end the endless conflicts
+        # and remove it.
+        find "$pkg_dir/$pkg" -name charset.alias -exec rm -f {} +
+
+        # Remove libtool's '*.la' library files. This removes cross-build
+        # system conflicts that may arise. Build-systems change, libtool
+        # is getting deprecated, we don't want a package that depends on
+        # some package's '.la' files.
+        find "$pkg_dir/$pkg" -name '*.la' -exec rm -f {} +
+
+        log "$pkg" "Successfully built package"
+
+        run_hook post-build "$pkg" "$pkg_dir/$pkg"
+
+        # Create the manifest file early and make it empty.
+        # This ensures that the manifest is added to the manifest.
+        : > "$pkg_dir/$pkg/$pkg_db/$pkg/manifest"
+
+        # If the package contains '/etc', add a file called
+        # 'etcsums' to the manifest. See comment directly above.
+        [ -d "$pkg_dir/$pkg/etc" ] &&
+            : > "$pkg_dir/$pkg/$pkg_db/$pkg/etcsums"
+
+        pkg_strip    "$pkg"
+        pkg_manifest "$pkg"
+        pkg_fix_deps "$pkg"
+        pkg_manifest "$pkg"
+        pkg_etcsums  "$pkg"
+        pkg_tar      "$pkg"
+
+        # Install only dependencies of passed packages.
+        # Skip this check if this is a package update.
+        contains "$explicit" "$pkg" && [ -z "$pkg_update" ] && continue
+
+        log "$pkg" "Needed as a dependency or has an update, installing"
+
+        (CPT_FORCE=1 cpt-install "$pkg")
+    done
+
+    # End here as this was a system update and all packages have been installed.
+    [ "$pkg_update" ] && return
+
+    log "Successfully built package(s)"
+
+    # Turn the explicit packages into a 'list'.
+    # See [1] at top of script.
+    # shellcheck disable=2046,2086
+    set -- $explicit
+
+    # Only ask for confirmation if more than one package needs to be installed.
+    [ $# -gt 1 ] && prompt "Install built packages? [$*]" && {
+        cpt-install "$@"
+        return
+    }
+
+    log "Run 'cpt i $*' to install the package(s)"
+}
+
+pkg_checksums() {
+    # Generate checksums for packages.
+    repo_dir=$(pkg_find "$1")
+
+    [ -f "$repo_dir/sources" ] || return 0
+
+    while read -r src _ || [ "$src" ]; do
+        # Comment.
+        if [ -z "${src##\#*}" ]; then
+            continue
+
+        # File is local to the package.
+        elif [ -f "$repo_dir/$src" ]; then
+            src_path=$repo_dir/${src%/*}
+
+        # File is remote and was downloaded.
+        elif [ -f "$src_dir/$1/${src##*/}" ]; then
+            src_path=$src_dir/$1
+
+        # File is a git repository.
+        elif [ -z "${src##git+*}" ]; then continue
+
+        # Die here if source for some reason, doesn't exist.
+        else
+            die "$1" "Couldn't find source '$src'"
+        fi
+
+        # An easy way to get 'sha256sum' to print with the 'basename'
+        # of files is to 'cd' to the file's directory beforehand.
+        (cd "$src_path" && sh256 "${src##*/}") ||
+            die "$1" "Failed to generate checksums"
+    done < "$repo_dir/sources"
+}
+
+pkg_verify() {
+    # Verify all package checksums. This is achieved by generating a new set of
+    # checksums and then comparing those with the old set.
+    verify_cmd="NR==FNR{a[\$1];next}/^git  .*/{next}!((\$1)in a){exit 1}"
+
+    for pkg; do
+        repo_dir=$(pkg_find "$pkg")
+        [ -f "$repo_dir/sources" ] || continue
+
+        pkg_checksums "$pkg" | awk "$verify_cmd" - "$repo_dir/checksums" || {
+        log "$pkg" "Checksum mismatch"
+
+        # Instead of dying above, log it to the terminal. Also define a
+        # variable so we *can* die after all checksum files have been
+        # checked.
+        mismatch="$mismatch$pkg "
+    } done
+
+    [ -z "$mismatch" ] || die "Checksum mismatch with: ${mismatch% }"
+}
+
+pkg_conflicts() {
+    # Check to see if a package conflicts with another.
+    log "$1" "Checking for package conflicts"
+
+    # Filter the tarball's manifest and select only files
+    # and any files they resolve to on the filesystem
+    # (/bin/ls -> /usr/bin/ls).
+    while read -r file; do
+        case $file in */) continue; esac
+
+        # Use $CPT_ROOT in filename so that we follow its symlinks.
+        file=$CPT_ROOT/${file#/}
+
+        # We will only follow the symlinks of the directories, so we reserve the
+        # directory name in this 'dirname' value. If we cannot find it in the
+        # system, we don't need to make this much more complex by trying so hard
+        # to find it. Simply use the original directory name.
+        dirname="$(_readlinkf "${file%/*}" 2>/dev/null)" ||
+            dirname="${file%/*}"
+
+
+        # Combine the dirname and file values, and print them into the
+        # temporary manifest to be parsed.
+        printf '%s/%s\n' "${dirname#$CPT_ROOT}" "${file##*/}"
+
+    done < "$tar_dir/$1/$pkg_db/$1/manifest" > "$CPT_TMPDIR/$pid/manifest"
+
+    p_name=$1
+
+    # Generate a list of all installed package manifests
+    # and remove the current package from the list.
+    # shellcheck disable=2046,2086
+    set -- $(set +f; pop "$sys_db/$p_name/manifest" from "$sys_db"/*/manifest)
+
+    [ -s "$CPT_TMPDIR/$pid/manifest" ] || return 0
+
+    # In rare cases where the system only has one package installed
+    # and you are reinstalling that package, grep will try to read from
+    # standard input if we continue here.
+    #
+    # Also, if we don't have any packages installed grep will give an
+    # error. This will not cause the installation to fail, but we don't
+    # need to check for conflicts if that's the case anyway. If we have
+    # only zero packages or one package, just stop wasting time and continue
+    # with the installation.
+    [ "$1" ] && [ -f "$1" ] || return 0
+
+    # Store the list of found conflicts in a file as we will be using the
+    # information multiple times. Storing it in the cache dir allows us
+    # to be lazy as they'll be automatically removed on script end.
+    "$grep" -Fxf "$CPT_TMPDIR/$pid/manifest" -- "$@" > "$CPT_TMPDIR/$pid/conflict" ||:
+
+
+    # Enable alternatives automatically if it is safe to do so.
+    # This checks to see that the package that is about to be installed
+    # doesn't overwrite anything it shouldn't in '/var/db/cpt/installed'.
+    "$grep" -q ":/var/db/cpt/installed/" "$CPT_TMPDIR/$pid/conflict" ||
+        choice_auto=1
+
+    # Use 'grep' to list matching lines between the to
+    # be installed package's manifest and the above filtered
+    # list.
+    if [ "$CPT_CHOICE" != 0 ] && [ "$choice_auto" = 1 ]; then
+
+        # This is a novel way of offering an "alternatives" system.
+        # It is entirely dynamic and all "choices" are created and
+        # destroyed on the fly.
+        #
+        # When a conflict is found between two packages, the file
+        # is moved to a directory called "choices" and its name
+        # changed to store its parent package and its intended
+        # location.
+        #
+        # The package's manifest is then updated to reflect this
+        # new location.
+        #
+        # The 'cpt choices' command parses this directory and
+        # offers you the CHOICE of *swapping* entries in this
+        # directory for those on the filesystem.
+        #
+        # The choices command does the same thing we do here,
+        # it rewrites manifests and moves files around to make
+        # this work.
+        #
+        # Pretty nifty huh?
+        while IFS=: read -r _ con; do
+            printf '%s\n' "Found conflict $con"
+
+            # Create the "choices" directory inside of the tarball.
+            # This directory will store the conflicting file.
+            mkdir -p "$tar_dir/$p_name/${cho_dir:=var/db/cpt/choices}"
+
+            # Construct the file name of the "db" entry of the
+            # conflicting file. (pkg_name>usr>bin>ls)
+            con_name=$(printf %s "$con" | sed 's|/|>|g')
+
+            # Move the conflicting file to the choices directory
+            # and name it according to the format above.
+            mv -f "$tar_dir/$p_name/$con" \
+                  "$tar_dir/$p_name/$cho_dir/$p_name$con_name" 2>/dev/null || {
+                log "File must be in ${con%/*} and not a symlink to it"
+                log "This usually occurs when a binary is installed to"
+                log "/sbin instead of /usr/bin (example)"
+                log "Before this package can be used as an alternative,"
+                log "this must be fixed in $p_name. Contact the maintainer"
+                die "by checking 'git log' or by running 'cpt-maintainer'"
+            }
+        done < "$CPT_TMPDIR/$pid/conflict"
+
+        # Rewrite the package's manifest to update its location
+        # to its new spot (and name) in the choices directory.
+        pkg_manifest "$p_name" "$tar_dir" 2>/dev/null
+
+    elif [ -s "$CPT_TMPDIR/$pid/conflict" ]; then
+        log "Package '$p_name' conflicts with another package" "" "!>"
+        log "Run 'CPT_CHOICE=1 cpt i $p_name' to add conflicts" "" "!>"
+        die "as alternatives."
+    fi
+}
+
+pkg_swap() {
+    # Swap between package alternatives.
+    pkg_list "$1" >/dev/null
+
+    alt=$(printf %s "$1$2" | sed 's|/|>|g')
+    cd "$sys_db/../choices"
+
+    [ -f "$alt" ] || [ -h "$alt" ] ||
+        die "Alternative '$1 $2' doesn't exist"
+
+    if [ -f "$2" ]; then
+        # Figure out which package owns the file we are going to swap for
+        # another package's.
+        #
+        # Print the full path to the manifest file which contains
+        # the match to our search.
+
+        pkg_owns=$(pkg_owner -lFx "$2") ||
+            die "File '$2' exists on filesystem but isn't owned"
+
+        log "Swapping '$2' from '$pkg_owns' to '$1'"
+
+        # Convert the current owner to an alternative and rewrite
+        # its manifest file to reflect this. We then resort this file
+        # so no issues arise when removing packages.
+        cp  -Pf "$CPT_ROOT/$2" "$pkg_owns>${alt#*>}"
+        sed "s#^$(regesc "$2")\$#${PWD#$CPT_ROOT}/$pkg_owns>${alt#*>}#" \
+            "../installed/$pkg_owns/manifest" |
+            sort -r -o "../installed/$pkg_owns/manifest"
+    fi
+
+    # Convert the desired alternative to a real file and rewrite
+    # the manifest file to reflect this. The reverse of above.
+    mv -f "$alt" "$CPT_ROOT/$2"
+    sed "s#^${PWD#$CPT_ROOT}/$(regesc "$alt")\$#$2#" "../installed/$1/manifest" |
+        sort -r -o "../installed/$1/manifest"
+}
+
+pkg_etc() {
+    [ -d "$tar_dir/$pkg_name/etc" ] || return 0
+
+    (cd "$tar_dir/$pkg_name"
+
+    # Create all directories beforehand.
+    find etc -type d | while read -r dir; do
+        mkdir -p "$CPT_ROOT/$dir"
+    done
+
+    # Handle files in /etc/ based on a 3-way checksum check.
+    find etc ! -type d | while read -r file; do
+        { sum_new=$(sh256 "$file")
+          sum_sys=$(cd "$CPT_ROOT/"; sh256 "$file")
+          sum_old=$("$grep" "$file$" "$mak_dir/c"); } 2>/dev/null ||:
+
+        log "$pkg_name" "Doing 3-way handshake for $file"
+        printf '%s\n' "Previous: ${sum_old:-null}"
+        printf '%s\n' "System:   ${sum_sys:-null}"
+        printf '%s\n' "New:      ${sum_new:-null}"
+
+        # Use a case statement to easily compare three strings at
+        # the same time. Pretty nifty.
+        case ${sum_old:-null}${sum_sys:-null}${sum_new} in
+            # old = Y, sys = X, new = Y
+            "${sum_new}${sum_sys}${sum_old}")
+                log "Skipping $file"
+                continue
+            ;;
+
+            # old = X, sys = X, new = X
+            # old = X, sys = Y, new = Y
+            # old = X, sys = X, new = Y
+            "${sum_old}${sum_old}${sum_old}"|\
+            "${sum_old:-null}${sum_sys}${sum_sys}"|\
+            "${sum_sys}${sum_old}"*)
+                log "Installing $file"
+                new=
+            ;;
+
+            # All other cases.
+            *)
+                warn "$pkg_name" "saving /$file as /$file.new" "->"
+                new=.new
+            ;;
+        esac
+
+        cp -fPp "$file"  "$CPT_ROOT/${file}${new}"
+        chown root:root "$CPT_ROOT/${file}${new}" 2>/dev/null
+    done) ||:
+}
+
+pkg_remove() {
+    # Remove a package and all of its files. The '/etc' directory
+    # is handled differently and configuration files are *not*
+    # overwritten.
+    pkg_list "$1" >/dev/null || return
+
+    # Make sure that nothing depends on this package.
+    [ "$CPT_FORCE" = 1 ] || {
+        log "$1" "Checking for reverse dependencies"
+
+        (cd "$sys_db"; set +f; grep -lFx "$1" -- */depends) &&
+            die "$1" "Can't remove package, others depend on it"
+    }
+    # Block being able to abort the script with 'Ctrl+C' during removal.
+    # Removes all risk of the user aborting a package removal leaving
+    # an incomplete package installed.
+    trap_set block
+
+    if [ -x "$sys_db/$1/pre-remove" ]; then
+        log "$1" "Running pre-remove script"
+        "$sys_db/$1/pre-remove" ||:
+    fi
+
+    # Create a temporary list of all directories, so we don't accidentally
+    # remove anything from packages that create empty directories for a
+    # purpose (such as baselayout).
+    manifest_list="$(set +f; pop "$sys_db/$1/manifest" from "$sys_db/"*/manifest)"
+    # shellcheck disable=2086
+    [ "$manifest_list" ] && grep -h '/$' $manifest_list | sort -ur > "$mak_dir/dirs"
+
+    run_hook pre-remove "$1" "$sys_db/$1" root
+
+    while read -r file; do
+        # The file is in '/etc' skip it. This prevents the package
+        # manager from removing user edited configuration files.
+        [ "${file##/etc/*}" ] || continue
+
+        if [ -d "$CPT_ROOT/$file" ]; then
+            "$grep" -Fxq "$file" "$mak_dir/dirs" 2>/dev/null && continue
+            rmdir "$CPT_ROOT/$file" 2>/dev/null || continue
+        else
+            rm -f "$CPT_ROOT/$file"
+        fi
+    done < "$sys_db/$1/manifest"
+
+    # Reset 'trap' to its original value. Removal is done so
+    # we no longer need to block 'Ctrl+C'.
+    trap_set cleanup
+
+    run_hook post-remove "$1" "$CPT_ROOT/" root
+
+    log "$1" "Removed successfully"
+}
+
+pkg_install() {
+    # Install a built package tarball.
+
+    # Install can also take the full path to a tarball.
+    # We don't need to check the repository if this is the case.
+    if [ -f "$1" ] && [ -z "${1%%*.tar*}" ] ; then
+        tar_file=$1
+        pkg_name=${1##*/}
+        pkg_name=${pkg_name%#*}
+
+    else
+        pkg_cache "$1" ||
+            die "package has not been built, run 'cpt b pkg'"
+
+        pkg_name=$1
+    fi
+
+    mkdir -p "$tar_dir/$pkg_name"
+    log "$pkg_name" "Extracting $tar_file"
+
+    # Extract the tarball to catch any errors before installation begins.
+    decompress "$tar_file" | "$tar" xf - -C "$tar_dir/$pkg_name"
+
+    [ -f "$tar_dir/$pkg_name/$pkg_db/$pkg_name/manifest" ] ||
+        die "'${tar_file##*/}' is not a valid CPT package"
+
+    # Ensure that the tarball's manifest is correct by checking that
+    # each file and directory inside of it actually exists.
+    [ "$CPT_FORCE" != 1 ] && log "$pkg_name" "Checking package manifest" &&
+        while read -r line; do
+            # Skip symbolic links
+            [ -h "$tar_dir/$pkg_name/$line" ] ||
+            [ -e "$tar_dir/$pkg_name/$line" ] || {
+        log "File $line missing from tarball but mentioned in manifest" "" "!>"
+        TARBALL_FAIL=1
+        }
+    done < "$tar_dir/$pkg_name/$pkg_db/$pkg_name/manifest"
+    [ "$TARBALL_FAIL" ] && {
+        log "You can still install this package by setting CPT_FORCE variable"
+        die "$pkg_name" "Missing files in manifest"
+    }
+
+    log "$pkg_name" "Checking that all dependencies are installed"
+
+    # Make sure that all run-time dependencies are installed prior to
+    # installing the package.
+    [ -f "$tar_dir/$pkg_name/$pkg_db/$pkg_name/depends" ] &&
+    [ "$CPT_FORCE" != 1 ] &&
+        while read -r dep dep_type || [ "$dep" ]; do
+            [ "${dep##\#*}" ] || continue
+            [ "$dep_type" ]   || pkg_list "$dep" >/dev/null ||
+                install_dep="$install_dep'$dep', "
+        done < "$tar_dir/$pkg_name/$pkg_db/$pkg_name/depends"
+
+    [ "$install_dep" ] && die "$1" "Package requires ${install_dep%, }"
+
+    run_hook pre-install "$pkg_name" "$tar_dir/$pkg_name" root
+
+    pkg_conflicts "$pkg_name"
+
+    log "$pkg_name" "Installing package incrementally"
+
+    # Block being able to abort the script with Ctrl+C during installation.
+    # Removes all risk of the user aborting a package installation leaving
+    # an incomplete package installed.
+    trap_set block
+
+    # If the package is already installed (and this is an upgrade) make a
+    # backup of the manifest and etcsums files.
+    cp -f "$sys_db/$pkg_name/manifest" "$mak_dir/m" 2>/dev/null ||:
+    cp -f "$sys_db/$pkg_name/etcsums"  "$mak_dir/c" 2>/dev/null ||:
+
+    # This is repeated multiple times. Better to make it a function.
+    pkg_rsync() {
+        rsync "--chown=$USER:$USER" --chmod=Du-s,Dg-s,Do-s \
+              -WhHKa --no-compress --exclude /etc "${1:---}" \
+              "$tar_dir/$pkg_name/" "$CPT_ROOT/"
+    }
+
+    # Install the package by using 'rsync' and overwrite any existing files
+    # (excluding '/etc/').
+    pkg_rsync --info=progress2
+    pkg_etc
+
+    # Remove any leftover files if this is an upgrade.
+    "$grep" -vFxf "$sys_db/$pkg_name/manifest" "$mak_dir/m" 2>/dev/null |
+
+    while read -r file; do
+        file=$CPT_ROOT/$file
+
+        # Skip deleting some leftover files.
+        case $file in /etc/*) continue; esac
+
+        # Remove files.
+        if [ -f "$file" ] && [ ! -L "$file" ]; then
+            rm -f "$file"
+
+        # Remove file symlinks.
+        elif [ -h "$file" ] && [ ! -d "$file" ]; then
+            unlink "$file" ||:
+
+        # Skip directory symlinks.
+        elif [ -h "$file" ] && [ -d "$file" ]; then :
+
+        # Remove directories if empty.
+        elif [ -d "$file" ]; then
+            rmdir "$file" 2>/dev/null ||:
+        fi
+    done ||:
+
+    log "$pkg_name" "Verifying installation"
+    { pkg_rsync; pkg_rsync; } ||:
+
+    # Reset 'trap' to its original value. Installation is done so
+    # we no longer need to block 'Ctrl+C'.
+    trap_set cleanup
+
+    if [ -x "$sys_db/$pkg_name/post-install" ]; then
+        log "$pkg_name" "Running post-install script"
+        "$sys_db/$pkg_name/post-install" ||:
+    fi
+
+    run_hook post-install "$pkg_name" "$sys_db/$pkg_name" root
+
+    log "$pkg_name" "Installed successfully"
+}
+
+pkg_updates(){
+    # Check all installed packages for updates. So long as the installed
+    # version and the version in the repositories differ, it's considered
+    # an update.
+    [ "$CPT_FETCH" = 0 ] || pkg_fetch
+
+    log "Checking for new package versions"
+
+    set +f
+
+    for pkg in "$sys_db/"*; do
+        pkg_name=${pkg##*/}
+
+        # Read version and release information from the installed packages
+        # and repository.
+        read -r db_ver db_rel < "$pkg/version"
+        read -r re_ver re_rel < "$(pkg_find "$pkg_name")/version"
+
+        # Compare installed packages to repository packages.
+        [ "$db_ver-$db_rel" != "$re_ver-$re_rel" ] && {
+            printf '%s\n' "$pkg_name $db_ver-$db_rel ==> $re_ver-$re_rel"
+            outdated="$outdated$pkg_name "
+        }
+    done
+
+    set -f
+
+    # If the download option is specified only download the outdated packages
+    # and exit.
+    # shellcheck disable=2154
+    [ "$download_only" = 1 ] && {
+        log "Only sources for the packages will be acquired"
+        prompt || exit 0
+
+        for pkg in $outdated; do
+            pkg_sources "$pkg"
+        done
+
+        exit 0
+    }
+
+    contains "$outdated" cpt && {
+        log "Detected package manager update"
+        log "The package manager will be updated first"
+
+        prompt || exit 0
+
+        pkg_build cpt
+        cpt-install cpt
+
+        log "Updated the package manager"
+        log "Re-run 'cpt update' to update your system"
+
+        exit 0
+    }
+
+    [ "$outdated" ] || {
+        log "Everything is up to date"
+        return
+    }
+
+    log "Packages to update: ${outdated% }"
+
+    # Tell 'pkg_build' to always prompt before build.
+    pkg_update=1
+
+    # Build all packages requiring an update.
+    # See [1] at top of script.
+    # shellcheck disable=2046,2086
+    {
+        pkg_order $outdated
+        pkg_build $order
+    }
+
+    log "Updated all packages"
+}
+
+pkg_clean() {
+    # Clean up on exit or error. This removes everything related
+    # to the build.
+    [ "$CPT_DEBUG" != 1 ] || return 0
+
+    # Block 'Ctrl+C' while cache is being cleaned.
+    trap_set block
+
+    # Remove temporary items.
+    rm -rf -- "${CPT_TMPDIR:=$cac_dir/proc}/$pid"
+}
+
+create_cache() {
+    # A temporary directory can be specified apart from the cache
+    # directory in order to build in a user specified directory.
+    # /tmp could be used in order to build on ram, useful on SSDs.
+    # The user can specify CPT_TMPDIR for this.
+    #
+    # Create the required temporary directories and set the variables
+    # which point to them.
+    mkdir -p "${tmp_dir:=${CPT_TMPDIR:=$cac_dir/proc}/$pid}"
+
+    # If an argument is given, skip the creation of other cache directories.
+    # This here makes shellcheck extremely angry, so I am globally disabling
+    # SC2119.
+    [ "$1" ] || mkdir -p "${mak_dir:=$tmp_dir/build}" \
+                         "${pkg_dir:=$tmp_dir/pkg}" \
+                         "${tar_dir:=$tmp_dir/export}"
+
+}
+
+# Local Variables:
+# mode: sh
+# End: