Skip to content

Commit e9d706e

Browse files
elezarelezar
committed
Add a hook to disable device node creation in a container
If required, this hook creates a modified params file (with ModifyDeviceFiles: 0) in a tmpfs and mounts this over /proc/driver/nvidia/params. This prevents device node creation when running tools such as nvidia-smi. In general the creation of these devices is cosmetic as a container does not have the required cgroup access for the devices. Signed-off-by: Evan Lezar <[email protected]>
1 parent fdcd250 commit e9d706e

File tree

5 files changed

+341
-0
lines changed

5 files changed

+341
-0
lines changed

‎cmd/nvidia-cdi-hook/commands/commands.go

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@ import (
2222
"github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-cdi-hook/chmod"
2323
symlinks "github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-cdi-hook/create-symlinks"
2424
"github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-cdi-hook/cudacompat"
25+
disabledevicenodemodification "github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-cdi-hook/disable-device-node-modification"
2526
ldcache "github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-cdi-hook/update-ldcache"
2627
"github.com/NVIDIA/nvidia-container-toolkit/internal/logger"
2728
)
@@ -34,6 +35,7 @@ func New(logger logger.Interface) []*cli.Command {
3435
symlinks.NewCommand(logger),
3536
chmod.NewCommand(logger),
3637
cudacompat.NewCommand(logger),
38+
disabledevicenodemodification.NewCommand(logger),
3739
}
3840
}
3941

‎cmd/nvidia-cdi-hook/disable-device-node-modification/disable-device-node-modification.go

Lines changed: 153 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,153 @@
1+
/**
2+
# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
3+
# SPDX-License-Identifier: Apache-2.0
4+
#
5+
# Licensed under the Apache License, Version 2.0 (the "License");
6+
# you may not use this file except in compliance with the License.
7+
# You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing, software
12+
# distributed under the License is distributed on an "AS IS" BASIS,
13+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
# See the License for the specific language governing permissions and
15+
# limitations under the License.
16+
**/
17+
18+
package disabledevicenodemodification
19+
20+
import (
21+
"bufio"
22+
"bytes"
23+
"errors"
24+
"fmt"
25+
"io"
26+
"os"
27+
"strings"
28+
29+
"github.com/urfave/cli/v2"
30+
31+
"github.com/NVIDIA/nvidia-container-toolkit/internal/logger"
32+
"github.com/NVIDIA/nvidia-container-toolkit/internal/oci"
33+
)
34+
35+
const (
36+
nvidiaDriverParamsPath = "/proc/driver/nvidia/params"
37+
)
38+
39+
type command struct {
40+
logger logger.Interface
41+
}
42+
43+
type options struct {
44+
containerSpec string
45+
}
46+
47+
// NewCommand constructs an disable-device-node-modification subcommand with the specified logger
48+
func NewCommand(logger logger.Interface) *cli.Command {
49+
c := command{
50+
logger: logger,
51+
}
52+
return c.build()
53+
}
54+
55+
func (m command) build() *cli.Command {
56+
cfg := options{}
57+
58+
c := cli.Command{
59+
Name: "disable-device-node-modification",
60+
Usage: "Ensure that the /proc/driver/nvidia/params file present in the container does not allow device node modifications.",
61+
Before: func(c *cli.Context) error {
62+
return m.validateFlags(c, &cfg)
63+
},
64+
Action: func(c *cli.Context) error {
65+
return m.run(c, &cfg)
66+
},
67+
}
68+
69+
c.Flags = []cli.Flag{
70+
&cli.StringFlag{
71+
Name: "container-spec",
72+
Hidden: true,
73+
Usage: "Specify the path to the OCI container spec. If empty or '-' the spec will be read from STDIN",
74+
Destination: &cfg.containerSpec,
75+
},
76+
}
77+
78+
return &c
79+
}
80+
81+
func (m command) validateFlags(c *cli.Context, cfg *options) error {
82+
return nil
83+
}
84+
85+
func (m command) run(c *cli.Context, cfg *options) error {
86+
// TODO: Do we need to prefix the driver root?
87+
hostNvidiaParamsFile, err := os.Open(nvidiaDriverParamsPath)
88+
if errors.Is(err, os.ErrNotExist) {
89+
return nil
90+
}
91+
if err != nil {
92+
return fmt.Errorf("failed to load params file: %w", err)
93+
}
94+
defer hostNvidiaParamsFile.Close()
95+
96+
s, err := oci.LoadContainerState(cfg.containerSpec)
97+
if err != nil {
98+
return fmt.Errorf("failed to load container state: %w", err)
99+
}
100+
101+
containerRoot, err := s.GetContainerRoot()
102+
if err != nil {
103+
return fmt.Errorf("failed to determined container root: %w", err)
104+
}
105+
106+
return m.updateNvidiaParamsFromReader(hostNvidiaParamsFile, containerRoot)
107+
}
108+
109+
func (m command) updateNvidiaParamsFromReader(r io.Reader, containerRootDirPath string) error {
110+
modifiedContents, err := m.getModifiedParamsFileContentsFromReader(r)
111+
if err != nil {
112+
return fmt.Errorf("failed to generate modified contents: %w", err)
113+
}
114+
if len(modifiedContents) == 0 {
115+
m.logger.Debugf("No modification required")
116+
return nil
117+
}
118+
119+
return createParamsFileInContainer(containerRootDirPath, modifiedContents)
120+
}
121+
122+
// getModifiedParamsFileContentsFromReader returns the contents of a modified params file from the specified reader.
123+
func (m command) getModifiedParamsFileContentsFromReader(r io.Reader) ([]byte, error) {
124+
var modified bytes.Buffer
125+
scanner := bufio.NewScanner(r)
126+
127+
var requiresModification bool
128+
for scanner.Scan() {
129+
line := scanner.Text()
130+
if strings.HasPrefix(line, "ModifyDeviceFiles: ") {
131+
if line == "ModifyDeviceFiles: 0" {
132+
m.logger.Debugf("Device node modification is already disabled")
133+
return nil, nil
134+
}
135+
if line == "ModifyDeviceFiles: 1" {
136+
line = "ModifyDeviceFiles: 0"
137+
requiresModification = true
138+
}
139+
}
140+
if _, err := modified.WriteString(line + "\n"); err != nil {
141+
return nil, fmt.Errorf("failed to create output buffer: %w", err)
142+
}
143+
}
144+
if err := scanner.Err(); err != nil {
145+
return nil, fmt.Errorf("failed to read params file: %w", err)
146+
}
147+
148+
if !requiresModification {
149+
return nil, nil
150+
}
151+
152+
return modified.Bytes(), nil
153+
}

‎cmd/nvidia-cdi-hook/disable-device-node-modification/disable-device-node-modification_test.go

Lines changed: 96 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,96 @@
1+
/**
2+
# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
3+
# SPDX-License-Identifier: Apache-2.0
4+
#
5+
# Licensed under the Apache License, Version 2.0 (the "License");
6+
# you may not use this file except in compliance with the License.
7+
# You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing, software
12+
# distributed under the License is distributed on an "AS IS" BASIS,
13+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
# See the License for the specific language governing permissions and
15+
# limitations under the License.
16+
**/
17+
18+
package disabledevicenodemodification
19+
20+
import (
21+
"bytes"
22+
"testing"
23+
24+
testlog "github.com/sirupsen/logrus/hooks/test"
25+
"github.com/stretchr/testify/require"
26+
)
27+
28+
func TestGetModifiedParamsFileContentsFromReader(t *testing.T) {
29+
logger, _ := testlog.NewNullLogger()
30+
testCases := map[string]struct {
31+
contents []byte
32+
expectedError error
33+
expectedContents []byte
34+
}{
35+
"no contents": {
36+
contents: nil,
37+
expectedError: nil,
38+
expectedContents: nil,
39+
},
40+
"other contents are ignored": {
41+
contents: []byte(`# Some other content
42+
that we don't care about
43+
`),
44+
expectedError: nil,
45+
expectedContents: nil,
46+
},
47+
"already zero requires no modification": {
48+
contents: []byte("ModifyDeviceFiles: 0"),
49+
expectedError: nil,
50+
expectedContents: nil,
51+
},
52+
"leading spaces require no modification": {
53+
contents: []byte(" ModifyDeviceFiles: 1"),
54+
},
55+
"Trailing spaces require no modification": {
56+
contents: []byte("ModifyDeviceFiles: 1 "),
57+
},
58+
"Not 1 require no modification": {
59+
contents: []byte("ModifyDeviceFiles: 11"),
60+
},
61+
"single line requires modification": {
62+
contents: []byte("ModifyDeviceFiles: 1"),
63+
expectedError: nil,
64+
expectedContents: []byte("ModifyDeviceFiles: 0\n"),
65+
},
66+
"single line with trailing newline requires modification": {
67+
contents: []byte("ModifyDeviceFiles: 1\n"),
68+
expectedError: nil,
69+
expectedContents: []byte("ModifyDeviceFiles: 0\n"),
70+
},
71+
"other content is maintained": {
72+
contents: []byte(`ModifyDeviceFiles: 1
73+
other content
74+
that
75+
is maintained`),
76+
expectedError: nil,
77+
expectedContents: []byte(`ModifyDeviceFiles: 0
78+
other content
79+
that
80+
is maintained
81+
`),
82+
},
83+
}
84+
85+
for description, tc := range testCases {
86+
t.Run(description, func(t *testing.T) {
87+
c := command{
88+
logger: logger,
89+
}
90+
contents, err := c.getModifiedParamsFileContentsFromReader(bytes.NewReader(tc.contents))
91+
require.EqualValues(t, tc.expectedError, err)
92+
require.EqualValues(t, string(tc.expectedContents), string(contents))
93+
})
94+
}
95+
96+
}

‎cmd/nvidia-cdi-hook/disable-device-node-modification/params_linux.go

Lines changed: 63 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,63 @@
1+
//go:build linux
2+
3+
/**
4+
# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
5+
# SPDX-License-Identifier: Apache-2.0
6+
#
7+
# Licensed under the Apache License, Version 2.0 (the "License");
8+
# you may not use this file except in compliance with the License.
9+
# You may obtain a copy of the License at
10+
#
11+
# http://www.apache.org/licenses/LICENSE-2.0
12+
#
13+
# Unless required by applicable law or agreed to in writing, software
14+
# distributed under the License is distributed on an "AS IS" BASIS,
15+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16+
# See the License for the specific language governing permissions and
17+
# limitations under the License.
18+
**/
19+
20+
package disabledevicenodemodification
21+
22+
import (
23+
"fmt"
24+
"os"
25+
"path/filepath"
26+
27+
"github.com/opencontainers/runc/libcontainer/utils"
28+
"golang.org/x/sys/unix"
29+
)
30+
31+
func createParamsFileInContainer(containerRootDirPath string, contents []byte) error {
32+
tmpRoot, err := os.MkdirTemp("", "nvct-empty-dir*")
33+
if err != nil {
34+
return fmt.Errorf("failed to create temp root: %w", err)
35+
}
36+
37+
if err := createTmpFs(tmpRoot, len(contents)); err != nil {
38+
return fmt.Errorf("failed to create tmpfs mount for params file: %w", err)
39+
}
40+
41+
modifiedParamsFile, err := os.OpenFile(filepath.Join(tmpRoot, "nvct-params"), os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0444)
42+
if err != nil {
43+
return fmt.Errorf("failed to open modified params file: %w", err)
44+
}
45+
defer modifiedParamsFile.Close()
46+
47+
if _, err := modifiedParamsFile.Write(contents); err != nil {
48+
return fmt.Errorf("failed to write temporary params file: %w", err)
49+
}
50+
51+
err = utils.WithProcfd(containerRootDirPath, nvidiaDriverParamsPath, func(nvidiaDriverParamsFdPath string) error {
52+
return unix.Mount(modifiedParamsFile.Name(), nvidiaDriverParamsFdPath, "", unix.MS_BIND|unix.MS_RDONLY|unix.MS_NODEV|unix.MS_PRIVATE|unix.MS_NOSYMFOLLOW, "")
53+
})
54+
if err != nil {
55+
return fmt.Errorf("failed to mount modified params file: %w", err)
56+
}
57+
58+
return nil
59+
}
60+
61+
func createTmpFs(target string, size int) error {
62+
return unix.Mount("tmpfs", target, "tmpfs", 0, fmt.Sprintf("size=%d", size))
63+
}

‎cmd/nvidia-cdi-hook/disable-device-node-modification/params_other.go

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
//go:build !linux
2+
// +build !linux
3+
4+
/**
5+
# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
6+
# SPDX-License-Identifier: Apache-2.0
7+
#
8+
# Licensed under the Apache License, Version 2.0 (the "License");
9+
# you may not use this file except in compliance with the License.
10+
# You may obtain a copy of the License at
11+
#
12+
# http://www.apache.org/licenses/LICENSE-2.0
13+
#
14+
# Unless required by applicable law or agreed to in writing, software
15+
# distributed under the License is distributed on an "AS IS" BASIS,
16+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
17+
# See the License for the specific language governing permissions and
18+
# limitations under the License.
19+
**/
20+
21+
package disabledevicenodemodification
22+
23+
import "fmt"
24+
25+
func createParamsFileInContainer(containerRootDirPath string, contents []byte) error {
26+
return fmt.Errorf("not supported")
27+
}

0 commit comments

Comments
 (0)