Merge branch 'standardise-shell' into 'main'

Standardise shell

Closes #425

See merge request tpo/core/arti!533

.gitlab-ci.yml

@@ -25,7 +25,7 @@ shellcheck:
   stage: check
   image: koalaman/shellcheck-alpine
   script:
-    - apk add git
+    - apk add git bash
     - ./maint/shellcheck_all
 
 rust-latest:
@@ -124,6 +124,7 @@ build-repro:
   # If you upgrade this image, also change the one in docker_reproducible_build
   image: rust:1.59.0-alpine3.15
   script:
+    - apk add bash
     - ./maint/reproducible_build linux windows macos
   artifacts:
     paths:

crates/arti-config/README.md

@@ -8,3 +8,5 @@ implement [Tor](https://www.torproject.org/) in Rust.
 
 The project continues, but this particular crate is now superseded.
 This empty crate is published as a tombstone.
+
+License: MIT OR Apache-2.0

crates/fs-mistrust/README.md

@@ -114,13 +114,12 @@ You can adjust the [`Mistrust`] object to change what it permits:
 ```rust
 use fs_mistrust::Mistrust;
 
-let mut my_mistrust = Mistrust::new();
-
-// Assume that our home directory and its parents are all well-configured.
-my_mistrust.ignore_prefix("/home/doze/")?;
-
-// Assume that a given group will only contain trusted users.
-my_mistrust.trust_group_id(413);
+let my_mistrust = Mistrust::builder()
+    // Assume that our home directory and its parents are all well-configured.
+    .ignore_prefix("/home/doze/")
+    // Assume that a given group will only contain trusted users.
+    .trust_group(413)
+    .build()?;
 ```
 
 See [`Mistrust`] for more options.
@@ -193,6 +192,17 @@ systems, but we don't actually look at the details of any of these:
    * SELinux capabilities
    * POSIX (and other) ACLs.
 
+We use a somewhat inaccurate heuristic when we're checking the permissions
+of items _inside_ a target directory (using [`Verifier::check_content`] or
+[`CheckedDir`]): we continue to forbid untrusted-writeable directories and
+files, but we still allow readable ones, even if we insisted that the target
+directory itself was required to to be unreadable.  This is too permissive
+in the case of readable objects with hard links: if there is a hard link to
+the file somewhere else, then an untrusted user can read it.  It is also too
+restrictive in the case of writeable objects _without_ hard links: if
+untrusted users have no path to those objects, they can't actually write
+them.
+
 On Windows, we accept all file permissions and owners.
 
 We don't check for mount-points and the privacy of filesystem devices

crates/tor-persist/README.md

@@ -7,7 +7,7 @@ This crate is part of
 implement [Tor](https://www.torproject.org/) in Rust.
 
 For now, users should construct storage objects directly with (for
-example) [`FsStateMgr::from_path()`], but use them primarily via the
+example) [`FsStateMgr::from_path_and_mistrust()`], but use them primarily via the
 interfaces of the [`StateMgr`] trait.
 
 License: MIT OR Apache-2.0

maint/binary_size

-#!/bin/sh
+#!/bin/bash
 #
 # binary_size: Build arti with a given set of options, and
 # dump the binary size in a json format.
 
-set -eu
+set -euo pipefail
 
 ORIGDIR=$(pwd)
 TMPDIR=$(mktemp -d -t arti_binsize.XXXXXX)

maint/changed_crates

-#!/bin/sh
+#!/bin/bash
+
+set -euo pipefail
 
 TOP=$(dirname "$0")/..
 
-TAG="$1"
+TAG="${1:-}"
 
 if [ -z "$TAG" ]; then
     echo "You need to give a git revision as an argument."

maint/check_licenses

 #!/bin/bash
 
-set -e
+set -euo pipefail
 
 # A list of the licenses that we currently allow in our code.
 #

maint/coverage

@@ -22,7 +22,7 @@ Notes:
 EOF
 }
 
-set -e
+set -euo pipefail
 
 TOPDIR=$(dirname "$0")/..
 cd "$TOPDIR"

maint/coverage_fuzz_corpora

 #!/bin/bash
 
-set -e
+set -euo pipefail
 
-if [ -z "$LLVM_PROFILE_FILE" ]; then
+if [ -z "${LLVM_PROFILE_FILE:-}" ]; then
     echo "This script is meant to be run inside with_coverage" >&2
     exit 1
 fi
@@ -12,32 +12,22 @@ coverage_dir=$(dirname "$LLVM_PROFILE_FILE")
 TOPDIR=$(realpath "$(dirname "$0")/..")
 cd "$TOPDIR"
 
-# for some reason, compiling with coverage is very slow, especially for curve25519-dalek,
-# and mixing normal runs and coverage runs trash the cache. Here we set an alternative
-# target directory so it's possible to reuse cached artifacts between coverage runs.
+# set an alternative target directory so it's possible to reuse cached artifacts between coverage 
+# runs of coverage and fuzzing.
 export CARGO_TARGET_DIR=${CARGO_TARGET_DIR:-target-coverage}
 
-# remove comments starting with #@ to run in parallel. This makes output very messy, uses a lot more
-# ram and make the load average go crazy, but it's also way faster both to compile (due to 
-# curve25519-dalek compilation being so slow, and it using a single core), and to run because it's
-# essentially monothreaded too.
 for d in ./crates/*/fuzz; do
-    #@{
-        pushd "$(dirname "$d")"
-        crate=$(basename "$(dirname "$d")")
-        mkdir -p "$TOPDIR/target/debug/$crate"
-        mkdir -p "$coverage_dir/$crate"
-        for fuzzer in $(cargo fuzz list); do
-            cargo fuzz coverage "$fuzzer"
-
-            # we copy binary and coverage data where with_coverage expect it to be
-            cp "target-coverage/x86_64-unknown-linux-gnu/release/$fuzzer" "$TOPDIR/target/debug/$crate/$fuzzer"
-            mv "fuzz/coverage/$fuzzer/raw" "$coverage_dir/$crate/$fuzzer"
-        done
-        popd
-    #@}&
+    pushd "$(dirname "$d")"
+    crate=$(basename "$(dirname "$d")")
+    mkdir -p "$TOPDIR/target/debug/$crate"
+    mkdir -p "$coverage_dir/$crate"
+    for fuzzer in $(cargo fuzz list); do
+	# disable sanitizer to work in stable. Also make curve25519-dalek compilation much faster
+        cargo fuzz coverage "$fuzzer" --sanitizer=none
+
+        # we copy binary and coverage data where with_coverage expect it to be
+        cp "target-coverage/x86_64-unknown-linux-gnu/release/$fuzzer" "$TOPDIR/target/debug/$crate/$fuzzer"
+        mv "fuzz/coverage/$fuzzer/raw" "$coverage_dir/$crate/$fuzzer"
+    done
+    popd
 done
-
-#@for d in ./crates/*/fuzz; do
-#@    wait
-#@done

maint/docker_reproducible_build

-#!/bin/sh
+#!/bin/bash
 #
 # This script runs as the top level of our reproducible build process.
 # It launches the actual build script inside a docker container.
 
+set -euo pipefail
+
 ## use a fixed image to not suffer from image retaging when newer rustc or
 ## alpine emerges. Increase shm size for the reasons described in
 ## reproducible_build
@@ -11,4 +13,4 @@
 ## the build-repro job in .gitlab-ci.yml
 exec docker run --rm -i -v "$(git rev-parse --show-toplevel)":/builds/arti \
 		-w /builds/arti --shm-size=512m rust:1.59.0-alpine3.15 \
-		./maint/reproducible_build "$@"
+		sh -c "apk add bash && ./maint/reproducible_build $*"

maint/downgrade_dependencies

-#!/bin/sh
+#!/bin/bash
 #
 # Downgrades every one of our dependencies in Cargo.lock to the
 # earliest version listed in our Cargo.toml files.  (And then
@@ -11,6 +11,8 @@
 # successfully with the versions listed in Cargo.lock, while declaring
 # support for versions of our dependencies that won't actually work.
 
+set -euo pipefail
+
 cargo +nightly update -Z minimal-versions
 cargo update \
       -p crc32fast \

maint/fuzz_it_all

 #!/bin/bash
 
-set -e
+set -euo pipefail
 
 echo "Using toolchain +${RUST_FUZZ_TOOLCHAIN:=nightly}. (Override with \$RUST_FUZZ_TOOLCHAIN)"
 

maint/hooks/pre-commit

-#!/bin/sh
+#!/bin/bash
 
-set -e
+set -euo pipefail
 
 echo '+cargo fmt --all -- --check'
 cargo fmt --all -- --check

maint/hooks/pre-push

-#!/bin/sh
+#!/bin/bash
 
-set -e
+set -euo pipefail
 
 echo '+cargo fmt --all -- --check'
 cargo fmt --all -- --check

maint/readmes

-#!/bin/sh
+#!/bin/bash
 
-set -e
+set -euo pipefail
 cd "$(dirname "$0")/.."
 
 for subcargo in crates/*/Cargo.toml ; do

maint/reproducible_build

-#!/bin/sh
+#!/bin/bash
 #
 # This script is run inside a docker container as part of our
 # reproducible build process.
 #
-set -xeu
+set -xeuo pipefail
 if [ ! -f /.dockerenv ]; then
     echo Not running inside Docker, build will probably not be reproducible
     echo Use docker_reproducible_build instead to get the right environment
@@ -103,6 +103,8 @@ EOF
 	mv /arti/target/x86_64-apple-darwin/release/arti "$here"/arti-macos
 fi
 
+git config --global --add safe.directory /arti
+
 set +x
 echo "branch       :" "$(git rev-parse --abbrev-ref HEAD)"
 echo "commit       :" "$(git rev-parse HEAD)"

maint/shellcheck_all

-#!/bin/sh
+#!/bin/bash
+
+set -euo pipefail
 
 git grep -P --line-number '^#! ?/bin/(:?ba)?sh\b' | sed -n 's/:1:[^:]*$//p' | xargs shellcheck

maint/with_coverage

 #!/bin/bash
 
-set -e
+set -euo pipefail
 
 SCRIPT_NAME=$(basename "$0")
 

tests/chutney/setup

 #!/bin/bash
 
-set -xe
+set -xeuo pipefail
 
 SCRIPT_NAME=$(basename "$0")
 
@@ -52,7 +52,7 @@ target="networks/$NETWORK"
 cd "$(git rev-parse --show-toplevel)"
 echo "target=$target" > tests/chutney/arti.run
 
-if [ -z "${CHUTNEY_PATH}" ]; then
+if [ -z "${CHUTNEY_PATH:-}" ]; then
     # CHUTNEY_PATH isn't set; try cloning or updating a local chutney.
     if [ -d chutney ]; then
 	(cd ./chutney && git pull)

tests/chutney/teardown

 #!/bin/bash
-set -xe
+set -xeuo pipefail
 
 cd "$(git rev-parse --show-toplevel)"
 
-if [ -z "${CHUTNEY_PATH}" ]; then
+if [ -z "${CHUTNEY_PATH:-}" ]; then
     # Use the default chutney path we set up before.
     CHUTNEY_PATH="$(pwd)/chutney"
     export CHUTNEY_PATH