From 8815334401adc36d913792b770a59de515a89855 Mon Sep 17 00:00:00 2001 From: Forza Date: Sun, 17 Dec 2023 14:42:12 +0100 Subject: [PATCH] Initial commit with cgexec and user_cgroup --- README.md | 116 +++++++++++++++++++++- cgexec | 199 ++++++++++++++++++++++++++++++++++++++ openrc/user_cgroups.confd | 13 +++ openrc/user_cgroups.initd | 61 ++++++++++++ user_cgroup | 13 +++ user_cgroup.sudoersd | 2 + 6 files changed, 401 insertions(+), 3 deletions(-) create mode 100755 cgexec create mode 100644 openrc/user_cgroups.confd create mode 100755 openrc/user_cgroups.initd create mode 100755 user_cgroup create mode 100644 user_cgroup.sudoersd diff --git a/README.md b/README.md index 6dfd8bf..490692c 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,115 @@ +# Linux Control Groups + +Linux Control Groups, commonly referred to as cgroups, is a feature in the Linux kernel that provides a way to organise and manage system resources for processes. It allows you to allocate and limit resources like CPU, memory, and I/O bandwidth among different processes or groups of processes. + +There are two versions of cgroups, the original version 1 and the `unified`, version 2. + # cgexec -cgexec is a Bash script that allows users to execute a command within a cgroup, providing control over resource limits such as CPU, I/O, and memory. - -https://wiki.tnonline.net/w/Linux/cgexec \ No newline at end of file +cgexec is a Bash script that allows users to execute a command within a cgroup, providing control over resource limits such as CPU, I/O, and memory. + +The script requires v2/unified cgroups to function. + +Documentation is available at . + +## Usage + +``` +Attaches a program to a unified cgroup with defined limits. + +Usage: cgexec [options] [cmd args] +Options: + -c cpu.weight (0-10000) Set CPU priority + -C cpu.max (1-100) Set max CPU time in percent + -i io.weight (1-10000) Set I/O weight + -m memory.high (0-max) Set soft memory limit + -M memory.max (0-max) Set hard memory limit + -g group Create or attach to existing cgroup. Default is to use an ephemeral group + -b path Use as cgroup root + +Requires Linux Control Groups v2 mounted at . +``` + +## Examples + +Execute a command with default settings +``` +cgexec echo "Hello, cgroups!" +``` + +Limit CPU and memory for a command +``` +cgexec -c 50 -m 1G my_command +``` + +Attach to an existing cgroup +``` + cgexec -g mygroup my_command +``` + +Use a custom cgroup root +``` +cgexec -b /sys/fs/cgroup/mysubsystem my_command +``` + +# Enable user cgroups +It is possible to allow normal users to create their own cgroups by creating an initial cgroup container that is owned by the user. + +First set up a container for the user +``` +mkdir -p /sys/fs/cgroup/user/forza/main +echo "+cpu +memory +io" > /sys/fs/cgroup/user/cgroup.subtree_control +echo "+cpu +memory +io" > /sys/fs/cgroup/user/forza/cgroup.subtree_control +chown forza:forza /sys/fs/cgroup/user/forza +chown forza:forza /sys/fs/cgroup/user/forza/cgroup.procs +chown forza:forza /sys/fs/cgroup/user/forza/cgroup.threads +chown -R forza:forza /sys/fs/cgroup/user/forza/main +chmod 775 /sys/fs/cgroup/user/forza +chmod 775 /sys/fs/cgroup/user/forza/main +``` +User `` can now create new nested cgroups in `/sys/fs/cgroup/user/forza` and attach processes to it. +The catch-22 of cgroups is that all programs by default reside in in the root `/sys/fs/cgroup/cgroup.procs`, and even though a user can write to its own cgroup, they can not remove themselves from the root cgroup. + +One solution is to let the user start a `screen` or `tmux` session and then use root/sudo to move it to the user's cgroup. + +Then as root, check what `pids` belong to the uaer session and add them to the user cgroup: +``` +# ps af -u forza +5309 pts/0 S+ 0:00 | \_ screen +5310 ? Ss 0:00 | \_ SCREEN +5311 pts/1 Ss 0:00 | \_ -/bin/bash +5483 pts/1 R+ 0:00 | \_ ps af -u forza + +# echo 5309 > /sys/fs/cgroup/user/forza/main/cgroup.procs +# echo 5310 > /sys/fs/cgroup/user/forza/main/cgroup.procs +# echo 5311 > /sys/fs/cgroup/user/forza/main/cgroup.procs +``` +Now, the user's screen session is in the `main` cgroup and the user can create additional cgroups and move processes started from within this session to that cgroup. +``` +## Helper utility +The `user_cgroup` helper utility can automate the process of setting of user cgroups. + +- `user_cgroup.initd` is an OpenRC init script that creates a cgroup hierarchy for specified users. +- `user_cgroup.confd` is the configuration file for OpenRC. +- `user_cgroup.sudoersd` is a sudo config file for allowing users to use the `user_cgroup` utility. +- `user_cgroup` small tool to move a uaer's shell to the user cgroup. + +The sudo file is optional, but allows a user to put the script in `~/.bash_profile` for automatic use. + +Once installed, a user can simply run `user_cgroup` to move itselslf to the user cgroup. Now it is possible for the user to create its own sub cgroups with `cgexec`. + +``` +# ./cgexec ls -l +* Creating cgroup: /sys/fs/cgroup/user/forza/cmd-GEP0 +* Executing: ls -l + +total 60 +-rwxr-xr-x 1 forza forza 4402 Dec 17 14:20 cgexec +-rw-r--r-- 1 forza forza 34580 Dec 16 19:02 LICENSE +drwxr-xr-x 1 forza forza 72 Dec 16 21:16 openrc +-rw-r--r-- 1 forza forza 4106 Dec 17 13:57 README.md +-rwxr-xr-x 1 forza forza 315 Dec 17 13:37 user_cgroup +-rw-r--r-- 1 forza forza 79 Dec 17 13:40 user_cgroup.sudoersd + +* Cleaning up cgroup /sys/fs/cgroup/user/forza/cmd-GEP0 +``` \ No newline at end of file diff --git a/cgexec b/cgexec new file mode 100755 index 0000000..ddda4af --- /dev/null +++ b/cgexec @@ -0,0 +1,199 @@ +#!/bin/bash + +# shellcheck shell=bash + +# cgexec - execute a command in a cgroup +# version 2023-12-16.0 +# +# Requires Linux Control Groups version v2 (unified) +# https://www.kernel.org/doc/Documentation/cgroup-v2.txt +# +# SPDX-License-Identifier: GPL-3.0-or-later +# Copyright 2023 Forza + +# Declare variables +declare -i -r self=$$ +declare -i cpu=0 +declare -i cpu_max=100 +declare -i io=1 +declare -i errorlevel=0 +declare -i cg_keep=0 +declare -i wait=10 +declare mem_high="max" +declare mem_max="max" +declare cg="" +declare cgcmd="" +declare group="" +declare -r cgmnt="$(findmnt --noheadings --output TARGET -t cgroup2)" + +# Set base to the cgroup v2 mount point +if [ "$USER" != "root" ] && [ -d "${cgmnt}/user/${USER}" ]; then + declare base="${cgmnt}/user/${USER}" +else + declare base="${cgmnt}" +fi + +help_text(){ +cat << END + +Attaches a program to a cgroup with defined limits. +Requires Linux Control Groups v2. + +Usage: cgexec [options] [cmd args] +Options: + -c cpu.weight (0-10000) Set CPU priority + -C cpu.max (1-100) Set max CPU time in percent + -i io.weight (1-10000) Set I/O weight + -m memory.high (0-max) Set soft memory limit + -M memory.max (0-max) Set hard memory limit + -g group Create or attach to existing cgroup. Default is to use an ephemeral group + -b path Use as cgroup root +END +} + +cleanup(){ + # Clean up and remove the cgroup + if [ $cg_keep -eq 1 ]; then + echo "* Not removing existing cgroup" + exit $errorlevel + fi + + # Move the script to the base cgroup + echo -e "\n* Cleaning up cgroup ${cg}" + + if [ "$USER" != "root" ]; then + if [ -w "${base}/main/cgroup.procs" ]; then + echo $self > "${base}/main/cgroup.procs" + fi + else + if [ -w "${base}/cgroup.procs" ]; then + echo -e "\n* Cleaning up cgroup ${cg}" + echo $self > "${base}/cgroup.procs" + fi + fi + + # Attempt to remove the cgroup + while [ $wait -gt 0 ]; do + rmdir "${cg}" >/dev/null 2>&1 + if [ ! -d "${cg}" ]; then + break + else + echo -n "." + sleep 1 + ((wait--)) + fi + done + if [ $wait -eq 0 ]; then + echo "* Command exited but ${cg}/cgroup.procs is not empty" + echo "* Not removing ${cg}" + fi + exit $errorlevel +} + +# Check arguments +if [ $# -eq 0 ]; then + help_text + exit 1 +fi + +while getopts 'c:C:i:m:M:g:b:h' OPT; do + case "$OPT" in + i) + io="$OPTARG" + echo "* I/O weight = $OPTARG" + ;; + c) + cpu="$OPTARG" + echo "* CPU weight = $OPTARG" + ;; + C) + cpu_max="$OPTARG" + echo "* CPU max = $OPTARG %" + ;; + m) + mem_high="$OPTARG" + echo "* Memory High = $OPTARG" + ;; + M) + mem_max="$OPTARG" + echo "* Memory Max = $OPTARG" + ;; + g) + group="$OPTARG" + ;; + b) + base="$OPTARG" + echo "* Base = $OPTARG" + ;; + h) + help_text + exit 0 + ;; + *) + help_text + exit 1 + ;; + esac +done +shift $(( OPTIND - 1 )) +cgcmd="${1}" +shift + +# Check if cgcmd is a valid executable +if ! command -v "${cgcmd}" &> /dev/null; then + echo "Error: '${cgcmd}' is not a valid executable." + exit 1 +fi + +# Check for existing cgroup +if [ -n "${group}" ]; then + cg="${base}/${group}" + if [ ! -d "${cg}" ]; then + if [ ! -w "$(dirname "${cg}")" ]; then + echo "Error: You don't have write access to $(dirname "${cg}")." + echo "Cannot create ${cg}" + exit 1 + fi + mkdir "${cg}" + else + echo "* Using existing cgroup: ${cg}" + cg_keep=1 + if [ ! -w "${cg}/cgroup.procs" ]; then + echo "Error: You don't have write access to ${cg}." + exit 1 + fi + fi + echo $self > "${cg}/cgroup.procs" +else + # Create a temporary cgroup + cg="$(mktemp -d -p "${base}" cmd-XXXX)" + echo "* Creating cgroup: ${cg}" + echo $self > "${cg}/cgroup.procs" +fi + +# Set cgroup limits +test -w "${cg}/io.weight" && echo $io > "${cg}/io.weight" +test -w "${cg}/io.bfq.weight" && echo $io > "${cg}/io.bfq.weight" +test -w "${cg}/memory.high" && echo "${mem_high}" > "${cg}/memory.high" +test -w "${cg}/memory.max" && echo "${mem_max}" > "${cg}/memory.max" + +if [ $cpu -eq 0 ] && [ -w "${cg}/cpu.idle" ]; then + echo 1 > "${cg}/cpu.idle" +else + test -w "${cg}/cpu.weight" && echo $cpu > "${cg}/cpu.weight" +fi + +if [ $cpu_max -gt 0 ] && [ $cpu_max -lt 100 ] && [ -w "${cg}/cpu.max" ]; then + echo "$((cpu_max * 1000)) 100000" > "${cg}/cpu.max" +fi + +# Execute command +echo -e "* Executing: ${cgcmd} ${*} \n" +"${cgcmd}" "${@}" +errorlevel=$? +if [ $errorlevel -ne 0 ]; then + echo -e "\nWARNING: ${cgcmd} exited with return code $errorlevel" +fi + +# Cleanup before exiting +cleanup \ No newline at end of file diff --git a/openrc/user_cgroups.confd b/openrc/user_cgroups.confd new file mode 100644 index 0000000..df3b013 --- /dev/null +++ b/openrc/user_cgroups.confd @@ -0,0 +1,13 @@ +# user_cgroups configuration file + +# Enable a cgroup for these users. +users="user1 user2" + +# Enable controllers tgat users can use +#controllers="+cpu +memory +io" + +# Limit number of child cgroups that a user can create +#max_decendants=10 + +# Mount path for cgroup2 +#cgroup_path="/sys/fs/cgroup" \ No newline at end of file diff --git a/openrc/user_cgroups.initd b/openrc/user_cgroups.initd new file mode 100755 index 0000000..5bb904a --- /dev/null +++ b/openrc/user_cgroups.initd @@ -0,0 +1,61 @@ +#!/sbin/openrc-run + +# OpenRC init script for setting up cgroups for specified users +# Requires Linux Control Groups version v2 (unified) +# https://www.kernel.org/doc/Documentation/cgroup-v2.txt +# +# SPDX-License-Identifier: GPL-3.0-or-later +# Copyright 2023 Forza + +# shellcheck shell=sh +# shellcheck disable=SC2034 + +name="${RC_SVCNAME}" +description="User cgroups" +: "${controllers:="+cpu +memory +io"}" +: "${max_decendants:=10}" +: "${users:=""}" +: "${cgroup_path:="$(findmnt --noheadings --output TARGET -t cgroup2)"}" +base_path="${cgroup_path}/user" + +depend() { + need cgroups +} + +start() { + ebegin "Starting user cgroups" + + if [ -z "${cgroup_path}" ]; then + eerror "cgroup2 is not mounted." + return 1 + fi + + # Create base user cgrouo container + checkpath -d -m 0775 -o root:root "${base_path}" + + # Enable cgroup controllers + echo "${controllers}" > "${base_path}/cgroup.subtree_control" + + # Set up user cgroups + for user in ${users}; do + if ! id "${user}" > /dev/null 2>&1; then + eerror "User '${user}' does not exist. Skipping cgroup setup for this user." + continue + fi + if [ ! -d "${base_path}/${user}" ]; then + checkpath -d -m 0775 -o "${user}":"${user}" "${base_path}/${user}" + chown -R "${user}":"${user}" "${base_path}/${user}" + #find "${base_path}/${user}" -type f -exec chmod 664 {} + + fi + echo "$max_decendants" > "${base_path}/cgroup.max.descendants" + echo "${controllers}" > "${base_path}/${user}/cgroup.subtree_control" + done + einfo "User cgroups are mounted at ${base_path}/" + eend +} + +status() { + einfo "User cgroups are mounted at ${base_path}/" + einfo "The following users' cgroups are enabled:" + find "${base_path}" -mindepth 1 -maxdepth 1 -type d -exec basename {} \; +} \ No newline at end of file diff --git a/user_cgroup b/user_cgroup new file mode 100755 index 0000000..b7422d7 --- /dev/null +++ b/user_cgroup @@ -0,0 +1,13 @@ +#!/bin/sh +[ "root" != "$USER" ] && exec env -i sudo $0 $PPID "$(id -un)" +declare -r user_id=$1 +declare -r user_name=$2 +cgroup="/sys/fs/cgroup/user/${user_name}/main" + +[ "$user_name" = "root" ] && exit 0 + +if [ -w "${cgroup}" ]; then + echo "cgroup: ${cgroup}" + echo "${user_id}" > "${cgroup}/cgroup.procs" +fi +exit 0 diff --git a/user_cgroup.sudoersd b/user_cgroup.sudoersd new file mode 100644 index 0000000..fa52fd9 --- /dev/null +++ b/user_cgroup.sudoersd @@ -0,0 +1,2 @@ +# Allow user to use cgroups +forza ALL=(root) NOPASSWD: /opt/cgexec/user_cgroup