ccfos
/
huatuo
6
0
Fork 3
Code Issues Pull Requests Packages Projects Releases Wiki Activity

internal: cgroups: abstracted into an independent package 

- application is unaware of the type of the cgroup runtime on the physical machine
- implement the abstraction of the cgroup interface
- support runtime configuration
- support fetching metrics for each subsystem
- modify the metrics which using cgroups

Signed-off-by: Tonghao Zhang <tonghao@bamaicloud.com>
This commit is contained in:
Tonghao Zhang 2025-07-23 01:06:27 -04:00
parent cb913b6a6b
commit 19fdcfdce7
23 changed files with 530 additions and 529 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

29
cmd/huatuo-bamai/main.go
View File

@ -26,12 +26,12 @@ import (
_ "huatuo-bamai/core/events"
_ "huatuo-bamai/core/metrics"
"huatuo-bamai/internal/bpf"
"huatuo-bamai/internal/cgroups"
"huatuo-bamai/internal/conf"
"huatuo-bamai/internal/log"
"huatuo-bamai/internal/pod"
"huatuo-bamai/internal/services"
"huatuo-bamai/internal/storage"
"huatuo-bamai/internal/utils/cgrouputil"
"huatuo-bamai/internal/utils/pidutil"
"huatuo-bamai/pkg/tracing"
@ -49,13 +49,26 @@ func mainAction(ctx *cli.Context) error {
defer pidutil.RemovePidFile(ctx.App.Name)
// init cpu quota
host, err := cgrouputil.NewRuntimeCgroup(ctx.App.Name,
conf.Get().RuntimeCgroup.LimitInitCPU,
conf.Get().RuntimeCgroup.LimitMem)
cgr, err := cgroups.NewCgroupManager()
if err != nil {
return fmt.Errorf("new cgroup: %w", err)
return err
}
if err := cgr.NewRuntime(ctx.App.Name,
cgroups.ToSpec(
conf.Get().RuntimeCgroup.LimitInitCPU,
conf.Get().RuntimeCgroup.LimitMem,
),
); err != nil {
return fmt.Errorf("new runtime cgroup: %w", err)
}
defer func() {
_ = cgr.DeleteRuntime()
}()
if err := cgr.AddProc(uint64(os.Getpid())); err != nil {
return fmt.Errorf("cgroup add pid to cgroups.proc")
}
defer host.Delete()
// initialize the storage clients.
storageInitCtx := storage.InitContext{
@ -112,8 +125,8 @@ func mainAction(ctx *cli.Context) error {
services.Start(conf.Get().APIServer.TCPAddr, mgr, prom)
// update cpu quota
if err := host.UpdateCPU(conf.Get().RuntimeCgroup.LimitCPU); err != nil {
return fmt.Errorf("cg update cpu: %w", err)
if err := cgr.UpdateRuntime(cgroups.ToSpec(conf.Get().RuntimeCgroup.LimitCPU, 0)); err != nil {
return fmt.Errorf("update runtime: %w", err)
}
waitExit := make(chan os.Signal, 1)

43
core/autotracing/cpuidle.go
View File

@ -20,18 +20,17 @@ import (
"fmt"
"os"
"os/exec"
"path/filepath"
"runtime"
"strconv"
"strings"
"time"
"huatuo-bamai/internal/cgroups"
"huatuo-bamai/internal/conf"
"huatuo-bamai/internal/flamegraph"
"huatuo-bamai/internal/log"
"huatuo-bamai/internal/pod"
"huatuo-bamai/internal/storage"
"huatuo-bamai/internal/utils/cgrouputil"
"huatuo-bamai/pkg/tracing"
"huatuo-bamai/pkg/types"
)
@ -90,36 +89,22 @@ func readIntFromFile(filePath string) (int, error) {
}
func readCPUUsage(path string) (map[string]uint64, error) {
cpuacctPath := path + "/cpuacct.stat"
output, err := os.ReadFile(cpuacctPath)
// FIXME!!!
cgr, err := cgroups.NewCgroupManager()
if err != nil {
return nil, err
}
cpuUsage := make(map[string]uint64)
lines := strings.Split(string(output), "\n")
for _, line := range lines {
line = strings.TrimSpace(line)
if line == "" {
continue
}
parts := strings.Fields(line)
if len(parts) != 2 {
continue
}
key := parts[0]
valueStr := parts[1]
value, err := strconv.ParseUint(valueStr, 10, 64)
if err != nil {
return nil, err
}
cpuUsage[key] = value
usage, err := cgr.CpuUsage(path)
if err != nil {
return nil, err
}
cpuUsage["total"] = uint64(time.Now().UnixNano())
return cpuUsage, nil
return map[string]uint64{
"user": usage.User,
"system": usage.System,
"total": uint64(time.Now().UnixNano()),
}, nil
}
// UserHZtons because kernel USER_HZ = 100, the default value set to 10,000,000
@ -171,7 +156,7 @@ func updateCPUIdleIDMap(m cpuIdleIDMap) error {
for _, container := range containers {
_, ok := m[container.ID]
if ok {
m[container.ID].path = filepath.Join(cgrouputil.V1CpuPath(), container.CgroupSuffix)
m[container.ID].path = container.CgroupSuffix
m[container.ID].alive = true
} else {
temp := &containerCPUInfo{
@ -191,7 +176,7 @@ func updateCPUIdleIDMap(m cpuIdleIDMap) error {
deltaUser: 0,
deltaSys: 0,
timestamp: 0,
path: filepath.Join(cgrouputil.V1CpuPath(), container.CgroupSuffix),
path: container.CgroupSuffix,
alive: true,
}
m[container.ID] = temp

6
core/autotracing/dload.go
View File

@ -25,11 +25,11 @@ import (
"strings"
"time"
"huatuo-bamai/internal/cgroups/paths"
"huatuo-bamai/internal/conf"
"huatuo-bamai/internal/log"
"huatuo-bamai/internal/pod"
"huatuo-bamai/internal/storage"
"huatuo-bamai/internal/utils/cgrouputil"
"huatuo-bamai/pkg/tracing"
"huatuo-bamai/pkg/types"
@ -204,14 +204,14 @@ func updateIDMap(m dloadIDMap) error {
for _, container := range containers {
if _, ok := m[container.ID]; ok {
m[container.ID].name = container.CgroupSuffix
m[container.ID].path = cgrouputil.NewCPU().Path(container.CgroupSuffix)
m[container.ID].path = paths.Path("cpu", container.CgroupSuffix)
m[container.ID].container = container
m[container.ID].alive = true
continue
}
m[container.ID] = &containerDloadInfo{
path: cgrouputil.NewCPU().Path(container.CgroupSuffix),
path: paths.Path("cpu", container.CgroupSuffix),
name: container.CgroupSuffix,
container: container,
alive: true,

21
core/metrics/cpu_stat.go
View File

@ -19,9 +19,9 @@ import (
"sync"
"time"
"huatuo-bamai/internal/cgroups"
"huatuo-bamai/internal/log"
"huatuo-bamai/internal/pod"
"huatuo-bamai/internal/utils/cgrouputil"
"huatuo-bamai/pkg/metric"
"huatuo-bamai/pkg/tracing"
)
@ -46,9 +46,8 @@ type cpuStat struct {
}
type cpuStatCollector struct {
cpu *cgrouputil.CPU
cpuacct *cgrouputil.CPUAcct
mutex sync.Mutex
cgroup cgroups.Cgroup
mutex sync.Mutex
}
func init() {
@ -57,10 +56,14 @@ func init() {
}
func newCPUStat() (*tracing.EventTracingAttr, error) {
cgroup, err := cgroups.NewCgroupManager()
if err != nil {
return nil, err
}
return &tracing.EventTracingAttr{
TracingData: &cpuStatCollector{
cpu: cgrouputil.NewCPU(),
cpuacct: cgrouputil.NewCPUAcctDefault(),
cgroup: cgroup,
},
Flag: tracing.FlagMetric,
}, nil
@ -82,12 +85,12 @@ func (c *cpuStatCollector) cpuMetricUpdate(cpu *cpuStat, container *pod.Containe
return nil
}
raw, err := c.cpu.StatRaw(container.CgroupSuffix)
raw, err := c.cgroup.CpuStatRaw(container.CgroupSuffix)
if err != nil {
return err
}
usageTotal, err := c.cpuacct.Usage(container.CgroupSuffix)
usage, err := c.cgroup.CpuUsage(container.CgroupSuffix)
if err != nil {
return err
}
@ -99,7 +102,7 @@ func (c *cpuStatCollector) cpuMetricUpdate(cpu *cpuStat, container *pod.Containe
innerWaitSum: raw["inner_wait_sum"],
nrBursts: raw["nr_bursts"],
burstTime: raw["burst_time"],
cpuTotal: usageTotal,
cpuTotal: usage.Usage * 1000,
lastUpdate: now,
}

33
core/metrics/cpu_util.go
View File

@ -15,14 +15,15 @@
package collector
import (
"math"
"reflect"
"runtime"
"sync"
"time"
"huatuo-bamai/internal/cgroups"
"huatuo-bamai/internal/log"
"huatuo-bamai/internal/pod"
"huatuo-bamai/internal/utils/cgrouputil"
"huatuo-bamai/pkg/metric"
"huatuo-bamai/pkg/tracing"
)
@ -39,8 +40,7 @@ type cpuMetric struct {
type cpuUtilCollector struct {
cpuUtil []*metric.Data
cpuacct *cgrouputil.CPUAcct
cpu *cgrouputil.CPU
cgroup cgroups.Cgroup
// included struct for used in multi modules
hostCPUCount int
@ -55,6 +55,11 @@ func init() {
}
func newCPUUtil() (*tracing.EventTracingAttr, error) {
cgroup, err := cgroups.NewCgroupManager()
if err != nil {
return nil, err
}
return &tracing.EventTracingAttr{
TracingData: &cpuUtilCollector{
cpuUtil: []*metric.Data{
@ -62,9 +67,8 @@ func newCPUUtil() (*tracing.EventTracingAttr, error) {
metric.NewGaugeData("sys", 0, "sys for container and host", nil),
metric.NewGaugeData("total", 0, "total for container and host", nil),
},
cpuacct: cgrouputil.NewCPUAcctDefault(),
cpu: cgrouputil.NewCPU(),
hostCPUCount: runtime.NumCPU(),
cgroup: cgroup,
},
Flag: tracing.FlagMetric,
}, nil
@ -90,15 +94,14 @@ func (c *cpuUtilCollector) cpuMetricUpdate(cpuMetric *cpuMetric, container *pod.
cgroupPath = container.CgroupSuffix
}
usageTotal, err := c.cpuacct.Usage(cgroupPath)
stat, err := c.cgroup.CpuUsage(cgroupPath)
if err != nil {
return err
}
usageUsr, usageSys, err := c.cpuacct.Stat(cgroupPath)
if err != nil {
return err
}
usageTotal := stat.Usage
usageUsr := stat.User
usageSys := stat.System
// allow statistics 0
deltaTotal := usageTotal - cpuMetric.lastCPUTotal
@ -149,12 +152,18 @@ func (c *cpuUtilCollector) Update() ([]*metric.Data, error) {
}
for _, container := range containers {
count, err := c.cpu.CPUNum(container.CgroupSuffix)
cpuQuota, err := c.cgroup.CpuQuotaAndPeriod(container.CgroupSuffix)
if err != nil {
log.Infof("failed to get cpu count of %s, %v", container, err)
log.Infof("fetch container [%s] cpu quota and period: %v", container, err)
continue
}
if cpuQuota.Quota == math.MaxUint64 {
continue
}
count := int(cpuQuota.Quota / cpuQuota.Period)
containerMetric := container.LifeResouces("collector_cpu_util").(*cpuMetric)
if err := c.cpuMetricUpdate(containerMetric, container, count); err != nil {
log.Infof("failed to update cpu info of %s, %v", container, err)

4
core/metrics/loadavg.go
View File

@ -17,9 +17,9 @@ package collector
import (
"fmt"
"huatuo-bamai/internal/cgroups/paths"
"huatuo-bamai/internal/log"
"huatuo-bamai/internal/pod"
"huatuo-bamai/internal/utils/cgrouputil"
"huatuo-bamai/pkg/metric"
"huatuo-bamai/pkg/tracing"
@ -86,7 +86,7 @@ func (c *loadavgCollector) Update() ([]*metric.Data, error) {
}
for _, container := range containers {
stats, err := n.GetCpuLoad(container.Hostname, cgrouputil.NewCPU().Path(container.CgroupSuffix))
stats, err := n.GetCpuLoad(container.Hostname, paths.Path("cpu", container.CgroupSuffix))
if err != nil {
log.Debugf("failed to get %s load, %v", container, err)
continue

13
core/metrics/memory_events.go
View File

@ -17,15 +17,15 @@ package collector
import (
"fmt"
"huatuo-bamai/internal/cgroups"
"huatuo-bamai/internal/conf"
"huatuo-bamai/internal/pod"
"huatuo-bamai/internal/utils/cgrouputil"
"huatuo-bamai/pkg/metric"
"huatuo-bamai/pkg/tracing"
)
type memEventsCollector struct {
mem cgrouputil.Memory
cgroup cgroups.Cgroup
}
func init() {
@ -33,9 +33,14 @@ func init() {
}
func newMemEvents() (*tracing.EventTracingAttr, error) {
cgroup, err := cgroups.NewCgroupManager()
if err != nil {
return nil, err
}
return &tracing.EventTracingAttr{
TracingData: &memEventsCollector{
mem: *cgrouputil.NewMemory(),
cgroup: cgroup,
}, Flag: tracing.FlagMetric,
}, nil
}
@ -51,7 +56,7 @@ func (c *memEventsCollector) Update() ([]*metric.Data, error) {
metrics := []*metric.Data{}
for _, container := range containers {
raw, err := c.mem.EventsRaw(container.CgroupSuffix)
raw, err := c.cgroup.MemoryEventRaw(container.CgroupSuffix)
if err != nil {
return nil, err
}

12
core/metrics/memory_others.go
View File

@ -16,11 +16,9 @@ package collector
import (
"fmt"
"path/filepath"
"huatuo-bamai/internal/log"
"huatuo-bamai/internal/cgroups/paths"
"huatuo-bamai/internal/pod"
"huatuo-bamai/internal/utils/cgrouputil"
"huatuo-bamai/internal/utils/parseutil"
"huatuo-bamai/pkg/metric"
"huatuo-bamai/pkg/tracing"
@ -40,8 +38,8 @@ func newMemOthersCollector() (*tracing.EventTracingAttr, error) {
}, nil
}
func parseValueWithKey(path, key string) (uint64, error) {
filePath := filepath.Join(cgrouputil.V1MemoryPath(), path)
func parseValueWithKey(cgroupPath, cgroupFile, key string) (uint64, error) {
filePath := paths.Path("memory", cgroupPath, cgroupFile)
if key == "" {
return parseutil.ReadUint(filePath)
}
@ -84,11 +82,9 @@ func (c *memOthersCollector) Update() ([]*metric.Data, error) {
name: "local_direct_reclaim_time",
},
} {
path := filepath.Join(container.CgroupSuffix, t.path)
value, err := parseValueWithKey(path, t.key)
value, err := parseValueWithKey(container.CgroupSuffix, t.path, t.key)
if err != nil {
// FIXME: os maynot support this metric
log.Debugf("parse %s: %s", path, err)
continue
}

20
core/metrics/memory_stat.go
View File

@ -17,25 +17,33 @@ package collector
import (
"fmt"
"huatuo-bamai/internal/cgroups"
"huatuo-bamai/internal/conf"
"huatuo-bamai/internal/log"
"huatuo-bamai/internal/pod"
"huatuo-bamai/internal/utils/cgrouputil"
"huatuo-bamai/internal/utils/parseutil"
"huatuo-bamai/pkg/metric"
"huatuo-bamai/pkg/tracing"
)
type memStatCollector struct{}
type memStatCollector struct {
cgroup cgroups.Cgroup
}
func init() {
tracing.RegisterEventTracing("memory_stat", newMemStat)
}
func newMemStat() (*tracing.EventTracingAttr, error) {
cgroup, err := cgroups.NewCgroupManager()
if err != nil {
return nil, err
}
return &tracing.EventTracingAttr{
TracingData: &memStatCollector{},
Flag: tracing.FlagMetric,
TracingData: &memStatCollector{
cgroup: cgroup,
},
Flag: tracing.FlagMetric,
}, nil
}
@ -50,7 +58,7 @@ func (c *memStatCollector) Update() ([]*metric.Data, error) {
}
for _, container := range containers {
raw, err := parseutil.ParseRawKV(cgrouputil.V1MemoryPath() + container.CgroupSuffix + "/memory.stat")
raw, err := c.cgroup.MemoryStatRaw(container.CgroupSuffix)
if err != nil {
log.Infof("parse %s memory.stat %v", container.CgroupSuffix, err)
continue

4
core/metrics/runtime.go
View File

@ -17,8 +17,8 @@ package collector
import (
"time"
"huatuo-bamai/internal/cgroups"
"huatuo-bamai/internal/log"
"huatuo-bamai/internal/utils/cgrouputil"
"huatuo-bamai/internal/utils/parseutil"
"huatuo-bamai/pkg/metric"
"huatuo-bamai/pkg/tracing"
@ -111,7 +111,7 @@ func getMemoryMetric(p *procfs.Proc) []*metric.Data {
data[0] = metric.NewGaugeData("memory_vss", float64(status.VmSize)/1024, "memory vss", nil)
data[1] = metric.NewGaugeData("memory_rss", float64(status.VmRSS)/1024, "memory rss", nil)
rssI, err := parseutil.ReadUint(cgrouputil.V1MemoryPath() + "/huatuo-bamai/memory.usage_in_bytes")
rssI, err := parseutil.ReadUint(cgroups.RootFsFilePath("memory") + "/huatuo-bamai/memory.usage_in_bytes")
if err != nil {
log.Warnf("can't ParseUint, err: %v", err)
return nil

86
internal/cgroups/cgroups.go Normal file
View File

@ -0,0 +1,86 @@
// Copyright 2025 The HuaTuo Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package cgroups
import (
"fmt"
"path/filepath"
"huatuo-bamai/internal/cgroups/paths"
"huatuo-bamai/internal/cgroups/stats"
v1 "huatuo-bamai/internal/cgroups/v1"
v2 "huatuo-bamai/internal/cgroups/v2"
extcgroups "github.com/containerd/cgroups/v3"
"github.com/opencontainers/runtime-spec/specs-go"
)
var cpuPeriod uint64 = 100000
type Cgroup interface {
// Name returns the cgroup name.
Name() string
// New a runtime config instance.
NewRuntime(path string, spec *specs.LinuxResources) error
// Delete a runtime config
DeleteRuntime() error
// Update a runtime config
UpdateRuntime(spec *specs.LinuxResources) error
// Add pids to cgroup.procs
AddProc(pid uint64) error
// CpuUsage return cgroups user/system and total usage.
CpuUsage(path string) (*stats.CpuUsage, error)
// CpuStatRaw return cpu.stat raw data
CpuStatRaw(path string) (map[string]uint64, error)
// CpuQuotaAndPeriod cgroup quota and period
CpuQuotaAndPeriod(path string) (*stats.CpuQuota, error)
// MemoryStatRaw memory.stat
MemoryStatRaw(path string) (map[string]uint64, error)
// MemoryEventRaw memory.stat
MemoryEventRaw(path string) (map[string]uint64, error)
}
func NewCgroupManager() (Cgroup, error) {
switch extcgroups.Mode() {
case extcgroups.Legacy:
return v1.New()
case extcgroups.Hybrid, extcgroups.Unified:
return v2.New()
default:
return nil, fmt.Errorf("not supported")
}
}
func ToSpec(cpu float64, memory int64) *specs.LinuxResources {
spec := &specs.LinuxResources{}
if cpu != 0 {
quota := int64(cpu * float64(cpuPeriod))
spec.CPU = &specs.LinuxCPU{
Period: &cpuPeriod,
Quota: &quota,
}
}
if memory != 0 {
spec.Memory = &specs.LinuxMemory{Limit: &memory}
}
return spec
}
func RootFsFilePath(subsys string) string {
return filepath.Join(paths.RootfsDefaultPath, subsys)
}

25
internal/utils/cgrouputil/paths.go → internal/cgroups/paths/paths.go
View File

@ -12,25 +12,14 @@
// See the License for the specific language governing permissions and
// limitations under the License.
package cgrouputil
package paths
import (
"path/filepath"
)
import "path/filepath"
var v1RootfsDefaultPath = "/sys/fs/cgroup"
var RootfsDefaultPath = "/sys/fs/cgroup"
// CgroupRootFsFilePath join dir with cgroup rootfs
func CgroupRootFsFilePath(name string) string {
return filepath.Join(v1RootfsDefaultPath, name)
}
// V1CpuPath return the cpu dir in cgroup v1
func V1CpuPath() string {
return v1RootfsDefaultPath + "/cpu"
}
// V1MemoryPath return the memory dir in cgroup v1
func V1MemoryPath() string {
return v1RootfsDefaultPath + "/memory"
func Path(path ...string) string {
root := []string{RootfsDefaultPath}
return filepath.Join(append(root, path...)...)
}

27
internal/cgroups/stats/stats.go Normal file
View File

@ -0,0 +1,27 @@
// Copyright 2025 The HuaTuo Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package stats
// usec, microsecond
type CpuUsage struct {
Usage uint64
User uint64
System uint64
}
type CpuQuota struct {
Quota uint64
Period uint64
}

154
internal/cgroups/v1/cgroups.go Normal file
View File

@ -0,0 +1,154 @@
// Copyright 2025 The HuaTuo Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package v1
import (
"math"
"huatuo-bamai/internal/cgroups/paths"
"huatuo-bamai/internal/cgroups/stats"
"huatuo-bamai/internal/utils/parseutil"
extv1 "github.com/containerd/cgroups/v3/cgroup1"
"github.com/opencontainers/runtime-spec/specs-go"
)
var clockTicks = getClockTicks()
const microsecondsInSecond = 1000000
// a typed name for a cgroup subsystem
const (
subsysDevices = "devices"
subsysHugetlb = "hugetlb"
subsysFreezer = "freezer"
subsysPids = "pids"
subsysNetCLS = "net_cls"
subsysNetPrio = "net_prio"
subsysPerfEvent = "perf_event"
subsysCpuset = "cpuset"
subsysCpu = "cpu"
subsysCpuacct = "cpuacct"
subsysMemory = "memory"
subsysBlkio = "blkio"
subsysRdma = "rdma"
)
type CgroupV1 struct {
name string
cgroup extv1.Cgroup
}
func New() (*CgroupV1, error) {
return &CgroupV1{
name: "legacy",
}, nil
}
func (c *CgroupV1) Name() string {
return c.name
}
func (c *CgroupV1) NewRuntime(path string, spec *specs.LinuxResources) error {
cg, err := extv1.New(extv1.StaticPath(path), spec)
if err != nil {
return err
}
c.cgroup = cg
return nil
}
func (c *CgroupV1) DeleteRuntime() error {
rootfs, err := extv1.Load(extv1.RootPath)
if err != nil {
return err
}
if err := c.cgroup.MoveTo(rootfs); err != nil {
return err
}
return c.cgroup.Delete()
}
func (c *CgroupV1) UpdateRuntime(spec *specs.LinuxResources) error {
return c.cgroup.Update(spec)
}
func (c *CgroupV1) AddProc(pid uint64) error {
return c.cgroup.AddProc(pid)
}
func (c *CgroupV1) CpuUsage(path string) (*stats.CpuUsage, error) {
statPath := paths.Path(subsysCpu, path, "cpuacct.stat")
raw, err := parseutil.ParseRawKV(statPath)
if err != nil {
return nil, err
}
usagePath := paths.Path(subsysCpu, path, "cpuacct.usage")
usage, err := parseutil.ReadUint(usagePath)
if err != nil {
return nil, err
}
user := (raw["user"] * microsecondsInSecond) / clockTicks
system := (raw["system"] * microsecondsInSecond) / clockTicks
return &stats.CpuUsage{
User: user,
System: system,
Usage: usage / 1000,
}, nil
}
func (c *CgroupV1) CpuStatRaw(path string) (map[string]uint64, error) {
return parseutil.ParseRawKV(paths.Path(subsysCpu, path, "cpu.stat"))
}
func (c *CgroupV1) CpuQuotaAndPeriod(path string) (*stats.CpuQuota, error) {
periodPath := paths.Path(subsysCpu, path, "cpu.cfs_period_us")
period, err := parseutil.ReadUint(periodPath)
if err != nil {
return nil, err
}
quotaPath := paths.Path(subsysCpu, path, "cpu.cfs_quota_us")
quota, err := parseutil.ReadInt(quotaPath)
if err != nil {
return nil, err
}
if quota == -1 {
return &stats.CpuQuota{
Quota: math.MaxUint64,
Period: period,
}, nil
}
return &stats.CpuQuota{
Quota: uint64(quota),
Period: period,
}, nil
}
func (c *CgroupV1) MemoryStatRaw(path string) (map[string]uint64, error) {
return parseutil.ParseRawKV(paths.Path(subsysMemory, path, "memory.stat"))
}
func (c *CgroupV1) MemoryEventRaw(path string) (map[string]uint64, error) {
return parseutil.ParseRawKV(paths.Path(subsysMemory, path, "memory.events"))
}

2
internal/utils/cgrouputil/ticks.go → internal/cgroups/v1/ticks.go
View File

@ -28,7 +28,7 @@
limitations under the License.
*/
package cgrouputil
package v1
func getClockTicks() uint64 {
// The value comes from `C.sysconf(C._SC_CLK_TCK)`, and

129
internal/cgroups/v2/cgroups.go Normal file
View File

@ -0,0 +1,129 @@
// Copyright 2025 The HuaTuo Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package v2
import (
"fmt"
"math"
"strconv"
"huatuo-bamai/internal/cgroups/paths"
"huatuo-bamai/internal/cgroups/stats"
"huatuo-bamai/internal/utils/parseutil"
extv2 "github.com/containerd/cgroups/v3/cgroup2"
"github.com/opencontainers/runtime-spec/specs-go"
)
type CgroupV2 struct {
name string
cgroup *extv2.Manager
}
func New() (*CgroupV2, error) {
return &CgroupV2{
name: "unified",
}, nil
}
func (c *CgroupV2) Name() string {
return c.name
}
func (c *CgroupV2) NewRuntime(path string, spec *specs.LinuxResources) error {
m, err := extv2.NewSystemd("/", path+".slice", -1, extv2.ToResources(spec))
if err != nil {
return fmt.Errorf("cgroup2 new systemd: %w", err)
}
// enable cpu and memory cgroup controllers
if err := m.ToggleControllers([]string{"cpu", "memory"}, extv2.Enable); err != nil {
_ = m.DeleteSystemd()
return fmt.Errorf("cgroup2 enabling cpu and memory controllers: %w", err)
}
c.cgroup = m
return nil
}
func (c *CgroupV2) DeleteRuntime() error {
rootfs, err := extv2.LoadSystemd("/", "")
if err != nil {
return err
}
if err := c.cgroup.MoveTo(rootfs); err != nil {
return err
}
if err := c.cgroup.Delete(); err != nil {
return err
}
return c.cgroup.DeleteSystemd()
}
func (c *CgroupV2) UpdateRuntime(spec *specs.LinuxResources) error {
return c.cgroup.Update(extv2.ToResources(spec))
}
func (c *CgroupV2) AddProc(pid uint64) error {
return c.cgroup.AddProc(pid)
}
func (c *CgroupV2) CpuStatRaw(path string) (map[string]uint64, error) {
return parseutil.ParseRawKV(paths.Path(path, "cpu.stat"))
}
func (c *CgroupV2) CpuUsage(path string) (*stats.CpuUsage, error) {
raw, err := c.CpuStatRaw(path)
if err != nil {
return nil, err
}
return &stats.CpuUsage{
Usage: raw["usage_usec"],
User: raw["user_usec"],
System: raw["system_usec"],
}, nil
}
func (c *CgroupV2) CpuQuotaAndPeriod(path string) (*stats.CpuQuota, error) {
maxpath := paths.Path(path, "cpu.max")
maxQuota, period, err := parseutil.ParseKV(maxpath)
if err != nil {
return nil, err
}
if maxQuota == "max" {
return &stats.CpuQuota{Quota: math.MaxUint64, Period: period}, nil
}
quota, err := strconv.ParseUint(maxQuota, 10, 64)
if err != nil {
return nil, err
}
return &stats.CpuQuota{Quota: quota, Period: period}, nil
}
func (c *CgroupV2) MemoryStatRaw(path string) (map[string]uint64, error) {
return parseutil.ParseRawKV(paths.Path(path, "memory.stat"))
}
func (c *CgroupV2) MemoryEventRaw(path string) (map[string]uint64, error) {
return parseutil.ParseRawKV(paths.Path(path, "memory.events"))
}

4
internal/pod/container_css_default.go
View File

@ -31,8 +31,8 @@ import (
"time"
"huatuo-bamai/internal/bpf"
"huatuo-bamai/internal/cgroups"
"huatuo-bamai/internal/log"
"huatuo-bamai/internal/utils/cgrouputil"
"huatuo-bamai/pkg/types"
mapset "github.com/deckarep/golang-set"
@ -192,7 +192,7 @@ func cgroupCssNotify() {
rootSet := mapset.NewSet()
for _, subsys := range cgroupv1SubSysName {
root := cgrouputil.CgroupRootFsFilePath(subsys)
root := cgroups.RootFsFilePath(subsys)
realRoot, err := filepath.EvalSymlinks(root)
if err != nil {
continue

4
internal/pod/container_type.go
View File

@ -23,6 +23,8 @@ type ContainerType uint32
const (
ContainerTypeSidecar ContainerType = 1 << iota
ContainerTypeDaemonSet
ContainerTypeNode
ContainerTypeStatic
ContainerTypeNormal
ContainerTypeUnknown
_containerTypeAll
@ -35,6 +37,8 @@ var containerType2String = map[ContainerType]string{
ContainerTypeSidecar: "Sidecar",
ContainerTypeDaemonSet: "DaemonSet",
ContainerTypeNormal: "Normal",
ContainerTypeNode: "Node",
ContainerTypeStatic: "Static",
ContainerTypeUnknown: "Unknown",
}

79
internal/utils/cgrouputil/cpu.go
View File

@ -1,79 +0,0 @@
// Copyright 2025 The HuaTuo Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
/*
Copyright The containerd Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package cgrouputil
import (
"fmt"
"path/filepath"
"huatuo-bamai/internal/utils/parseutil"
)
// NewCPU new cpu obj with defalut rootfs
func NewCPU() *CPU {
return &CPU{
root: V1CpuPath(),
}
}
// CPU cgroup obj
type CPU struct {
root string
}
// Path join path with cgroup v1 rootfs
func (c *CPU) Path(path string) string {
return filepath.Join(c.root, path)
}
// StatRaw return kv slice in cpu.stat
func (c *CPU) StatRaw(path string) (map[string]uint64, error) {
return parseutil.ParseRawKV(filepath.Join(c.Path(path), "cpu.stat"))
}
// CPUCount return cgroup v1 cpu num
func (c *CPU) CPUNum(path string) (int, error) {
period, err := parseutil.ReadInt(filepath.Join(c.Path(path), "cpu.cfs_period_us"))
if err != nil {
return 0, err
}
if period == -1 {
return 0, fmt.Errorf("no limited")
}
quota, err := parseutil.ReadUint(filepath.Join(c.Path(path), "cpu.cfs_quota_us"))
if err != nil {
return 0, err
}
return int(quota / uint64(period)), nil
}

137
internal/utils/cgrouputil/cpuacct.go
View File

@ -1,137 +0,0 @@
// Copyright 2025 The HuaTuo Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
/*
Copyright The containerd Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package cgrouputil
import (
"bufio"
"fmt"
"os"
"path/filepath"
"strconv"
"strings"
"huatuo-bamai/internal/utils/parseutil"
)
const nanosecondsInSecond = 1000000000
var clockTicks = getClockTicks()
// NewCPUAcct new obj with rootfs
func NewCPUAcct(root string) *CPUAcct {
return &CPUAcct{
root: root,
}
}
// NewCPUAcctDefault new obj with default rootfs
func NewCPUAcctDefault() *CPUAcct {
return &CPUAcct{
root: V1CpuPath(),
}
}
// CPUAcct cgroup obj
type CPUAcct struct {
root string
}
// Path join file path
func (c *CPUAcct) Path(path string) string {
return filepath.Join(c.root, path)
}
// PercpuUsage return values in cpuacct.usage_percpu
func (c *CPUAcct) PercpuUsage(path string) ([]uint64, error) {
var usage []uint64
data, err := os.ReadFile(filepath.Join(c.Path(path), "cpuacct.usage_percpu"))
if err != nil {
return nil, err
}
for _, v := range strings.Fields(string(data)) {
u, err := strconv.ParseUint(v, 10, 64)
if err != nil {
return nil, err
}
usage = append(usage, u)
}
return usage, nil
}
// Usage return value in cpuacct.usage
func (c *CPUAcct) Usage(path string) (uint64, error) {
return parseutil.ReadUint(filepath.Join(c.Path(path), "cpuacct.usage"))
}
// Stat return user/kernel values in cpuacct.stat
func (c *CPUAcct) Stat(path string) (user, kernel uint64, err error) {
statPath := filepath.Join(c.Path(path), "cpuacct.stat")
f, err := os.Open(statPath)
if err != nil {
return 0, 0, err
}
defer f.Close()
var (
raw = make(map[string]uint64)
sc = bufio.NewScanner(f)
)
for sc.Scan() {
key, v, err := parseutil.ParseKV(sc.Text())
if err != nil {
return 0, 0, err
}
raw[key] = v
}
if err := sc.Err(); err != nil {
return 0, 0, err
}
for _, t := range []struct {
name string
value *uint64
}{
{
name: "user",
value: &user,
},
{
name: "system",
value: &kernel,
},
} {
v, ok := raw[t.name]
if !ok {
return 0, 0, fmt.Errorf("expected field %q but not found in %q", t.name, statPath)
}
*t.value = v
}
return (user * nanosecondsInSecond) / clockTicks, (kernel * nanosecondsInSecond) / clockTicks, nil
}

59
internal/utils/cgrouputil/memory.go
View File

@ -1,59 +0,0 @@
// Copyright 2025 The HuaTuo Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
/*
Copyright The containerd Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package cgrouputil
import (
"path/filepath"
"huatuo-bamai/internal/utils/parseutil"
)
// NewMemory new cpu obj with default rootfs
func NewMemory() *Memory {
return &Memory{
root: V1MemoryPath(),
}
}
// Memory cgroup obj
type Memory struct {
root string
}
// Path join path with cgroup v1 rootfs
func (c *Memory) Path(path string) string {
return filepath.Join(c.root, path)
}
// EventsRaw return kv slice in memory.events
func (c *Memory) EventsRaw(path string) (map[string]uint64, error) {
return parseutil.ParseRawKV(filepath.Join(c.Path(path), "memory.events"))
}

148
internal/utils/cgrouputil/runtime.go
View File

@ -1,148 +0,0 @@
// Copyright 2025 The HuaTuo Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package cgrouputil
import (
"fmt"
"os"
"huatuo-bamai/internal/log"
cgroups "github.com/containerd/cgroups/v3"
"github.com/containerd/cgroups/v3/cgroup1"
"github.com/containerd/cgroups/v3/cgroup2"
"github.com/opencontainers/runtime-spec/specs-go"
)
// RuntimeCgroup instance
type RuntimeCgroup struct {
cgv1 cgroup1.Cgroup
cgv2 *cgroup2.Manager
mode cgroups.CGMode
}
var runtimeCgroupPeriod uint64 = 100000
func newRuntimeCgroupV1(cgPath string, cgResources *specs.LinuxResources) (*RuntimeCgroup, error) {
cg, err := cgroup1.New(cgroup1.StaticPath(cgPath), cgResources)
if err != nil {
return nil, err
}
if err := cg.Add(cgroup1.Process{Pid: os.Getpid()}); err != nil {
_ = cg.Delete()
return nil, err
}
return &RuntimeCgroup{cgv1: cg, mode: cgroups.Legacy}, nil
}
func newRuntimeCgroupV2(cgPath string, cgResources *specs.LinuxResources) (*RuntimeCgroup, error) {
m, err := cgroup2.NewSystemd("/", cgPath+".slice", -1, cgroup2.ToResources(cgResources))
if err != nil {
return nil, fmt.Errorf("cgroup2 new systemd: %w", err)
}
// enable cpu and memory cgroup controllers
if err := m.ToggleControllers([]string{"cpu", "memory"}, cgroup2.Enable); err != nil {
_ = m.DeleteSystemd()
return nil, fmt.Errorf("cgroup2 enabling cpu and memory controllers: %w", err)
}
if err := m.AddProc(uint64(os.Getpid())); err != nil {
_ = m.DeleteSystemd()
return nil, fmt.Errorf("cgroup2 adding pids proc: %w", err)
}
log.Debugf("huatuo-bamai use cgroup path: %v", m)
return &RuntimeCgroup{cgv2: m, mode: cgroups.Unified}, nil
}
func runtimeCgroupMode(mode cgroups.CGMode) string {
switch mode {
case cgroups.Legacy:
return "legacy"
case cgroups.Unified:
return "unified"
case cgroups.Hybrid:
return "hybrid"
}
return "unavailable"
}
// NewRuntimeCgroup new instance
func NewRuntimeCgroup(cgPath string, cpu float64, mem int64) (*RuntimeCgroup, error) {
quota := int64(cpu * float64(runtimeCgroupPeriod))
cgResources := &specs.LinuxResources{
CPU: &specs.LinuxCPU{
Period: &runtimeCgroupPeriod,
Quota: &quota,
},
Memory: &specs.LinuxMemory{
Limit: &mem,
},
}
mode := cgroups.Mode()
switch mode {
case cgroups.Legacy:
return newRuntimeCgroupV1(cgPath, cgResources)
case cgroups.Unified:
return newRuntimeCgroupV2(cgPath, cgResources)
default:
return nil, fmt.Errorf("cgroup type(%s) not supported", runtimeCgroupMode(mode))
}
}
// Delete HostCgroup
func (host *RuntimeCgroup) Delete() {
// 1. move pids to cgroup rootfs temporarily
// 2. delete cgroups.
switch host.mode {
case cgroups.Legacy:
rootfs, _ := cgroup1.Load(cgroup1.RootPath)
_ = host.cgv1.MoveTo(rootfs)
_ = host.cgv1.Delete()
case cgroups.Unified:
rootfs, _ := cgroup2.LoadSystemd("/", "")
_ = host.cgv2.MoveTo(rootfs)
_ = host.cgv2.Delete()
_ = host.cgv2.DeleteSystemd()
}
}
// UpdateCPU update resource
func (host *RuntimeCgroup) UpdateCPU(cpu float64) error {
quota := int64(cpu * float64(runtimeCgroupPeriod))
cgResources := &specs.LinuxResources{
CPU: &specs.LinuxCPU{
Period: &runtimeCgroupPeriod,
Quota: &quota,
},
}
switch host.mode {
case cgroups.Legacy:
return host.cgv1.Update(cgResources)
case cgroups.Unified:
return host.cgv2.Update(cgroup2.ToResources(cgResources))
default:
return fmt.Errorf("cgroup type(%s) not supported", runtimeCgroupMode(host.mode))
}
}

20
internal/utils/parseutil/parseutils.go
View File

@ -42,7 +42,7 @@ func ReadInt(path string) (int64, error) {
return strconv.ParseInt(strings.TrimSpace(string(v)), 10, 64)
}
func ParseKV(raw string) (string, uint64, error) {
func parseKV(raw string) (string, uint64, error) {
parts := strings.Fields(raw)
switch len(parts) {
case 2:
@ -70,7 +70,7 @@ func ParseRawKV(path string) (map[string]uint64, error) {
)
for sc.Scan() {
key, v, err := ParseKV(sc.Text())
key, v, err := parseKV(sc.Text())
if err != nil {
return nil, err
}
@ -83,3 +83,19 @@ func ParseRawKV(path string) (map[string]uint64, error) {
return raw, nil
}
func ParseKV(path string) (string, uint64, error) {
f, err := os.Open(path)
if err != nil {
return "", 0, err
}
defer f.Close()
scanner := bufio.NewScanner(f)
scanner.Scan()
if err := scanner.Err(); err != nil {
return "", 0, err
}
return parseKV(scanner.Text())
}