Skip to content

Commit 8105fa0

Browse files
Refactor Remote Host Collection (#1633)
* refactor remote collectors * add remotecollect params struct * remove commented checkrbac function * removed unused function * add temp comments * refactor to not require RemoteCollect method per collector * removed unneeded param * removed unneeded param * more refactor * more refactor * remove unneeded function * remove debug print * fix analyzer results * move rbac to separate file * be more specific with rbac function name * fix imports * fix node list file * make k8s rest client config consistent with in cluster collection * add ctx and otel tracing * add test for allCollectedData * move runHostCollectorsInPod to spec instead of metadata * make generate * fix broken references to supportbundle metadata * add e2e tests * update loader tests * fix tests * fix hostos remote collector spec * update remoteHostCollectrs.yaml --------- Co-authored-by: Dexter Yan <[email protected]>
1 parent e7c07a7 commit 8105fa0

File tree

20 files changed

+558
-193
lines changed

20 files changed

+558
-193
lines changed

cmd/troubleshoot/cli/run.go

Lines changed: 9 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -110,7 +110,7 @@ func runTroubleshoot(v *viper.Viper, args []string) error {
110110
}
111111

112112
if interactive {
113-
if len(mainBundle.Spec.HostCollectors) > 0 && !util.IsRunningAsRoot() {
113+
if len(mainBundle.Spec.HostCollectors) > 0 && !util.IsRunningAsRoot() && !mainBundle.Spec.RunHostCollectorsInPod {
114114
fmt.Print(cursor.Show())
115115
if util.PromptYesNo(util.HOST_COLLECTORS_RUN_AS_ROOT_PROMPT) {
116116
fmt.Println("Exiting...")
@@ -184,7 +184,7 @@ func runTroubleshoot(v *viper.Viper, args []string) error {
184184
OutputPath: v.GetString("output"),
185185
Redact: v.GetBool("redact"),
186186
FromCLI: true,
187-
RunHostCollectorsInPod: mainBundle.Metadata.RunHostCollectorsInPod,
187+
RunHostCollectorsInPod: mainBundle.Spec.RunHostCollectorsInPod,
188188
}
189189

190190
nonInteractiveOutput := analysisOutput{}
@@ -199,7 +199,7 @@ func runTroubleshoot(v *viper.Viper, args []string) error {
199199

200200
if len(response.AnalyzerResults) > 0 {
201201
if interactive {
202-
if err := showInteractiveResults(mainBundle.Metadata.Name, response.AnalyzerResults, response.ArchivePath); err != nil {
202+
if err := showInteractiveResults(mainBundle.Name, response.AnalyzerResults, response.ArchivePath); err != nil {
203203
interactive = false
204204
}
205205
} else {
@@ -208,7 +208,7 @@ func runTroubleshoot(v *viper.Viper, args []string) error {
208208
}
209209

210210
if !response.FileUploaded {
211-
if appName := mainBundle.Metadata.Labels["applicationName"]; appName != "" {
211+
if appName := mainBundle.Labels["applicationName"]; appName != "" {
212212
f := `A support bundle for %s has been created in this directory
213213
named %s. Please upload it on the Troubleshoot page of
214214
the %s Admin Console to begin analysis.`
@@ -337,11 +337,8 @@ func loadSpecs(ctx context.Context, args []string, client kubernetes.Interface)
337337
APIVersion: "troubleshoot.sh/v1beta2",
338338
Kind: "SupportBundle",
339339
},
340-
Metadata: troubleshootv1beta2.SupportBundleMetadata{
341-
ObjectMeta: metav1.ObjectMeta{
342-
Name: "merged-support-bundle-spec",
343-
},
344-
RunHostCollectorsInPod: false,
340+
ObjectMeta: metav1.ObjectMeta{
341+
Name: "merged-support-bundle-spec",
345342
},
346343
}
347344

@@ -351,11 +348,11 @@ func loadSpecs(ctx context.Context, args []string, client kubernetes.Interface)
351348
sb := sb
352349
mainBundle = supportbundle.ConcatSpec(mainBundle, &sb)
353350
//check if sb has metadata and if it has RunHostCollectorsInPod set to true
354-
if !reflect.DeepEqual(sb.Metadata.ObjectMeta, metav1.ObjectMeta{}) && sb.Metadata.RunHostCollectorsInPod {
355-
enableRunHostCollectorsInPod = sb.Metadata.RunHostCollectorsInPod
351+
if !reflect.DeepEqual(sb.ObjectMeta, metav1.ObjectMeta{}) && sb.Spec.RunHostCollectorsInPod {
352+
enableRunHostCollectorsInPod = sb.Spec.RunHostCollectorsInPod
356353
}
357354
}
358-
mainBundle.Metadata.RunHostCollectorsInPod = enableRunHostCollectorsInPod
355+
mainBundle.Spec.RunHostCollectorsInPod = enableRunHostCollectorsInPod
359356

360357
for _, c := range kinds.CollectorsV1Beta2 {
361358
mainBundle.Spec.Collectors = util.Append(mainBundle.Spec.Collectors, c.Spec.Collectors)

config/crds/troubleshoot.sh_supportbundles.yaml

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2114,6 +2114,8 @@ spec:
21142114
type: string
21152115
exclude:
21162116
type: BoolString
2117+
image:
2118+
type: string
21172119
namespace:
21182120
type: string
21192121
podLaunchOptions:
@@ -20313,6 +20315,8 @@ spec:
2031320315
type: object
2031420316
type: object
2031520317
type: array
20318+
runHostCollectorsInPod:
20319+
type: boolean
2031620320
uri:
2031720321
description: URI optionally defines a location which is the source
2031820322
of this spec to allow updating of the spec at runtime

pkg/apis/troubleshoot/v1beta2/remote_collector_shared.go

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@ type RemoteCollectorMeta struct {
1616
type RemoteCPU struct {
1717
RemoteCollectorMeta `json:",inline" yaml:",inline"`
1818
}
19+
1920
type RemoteHostOS struct {
2021
RemoteCollectorMeta `json:",inline" yaml:",inline"`
2122
}

pkg/apis/troubleshoot/v1beta2/supportbundle_types.go

Lines changed: 4 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -20,11 +20,6 @@ import (
2020
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
2121
)
2222

23-
type SupportBundleMetadata struct {
24-
metav1.ObjectMeta `json:",inline" yaml:",inline"`
25-
RunHostCollectorsInPod bool `json:"runHostCollectorsInPod,omitempty" yaml:"runHostCollectorsInPod,omitempty"`
26-
}
27-
2823
// SupportBundleSpec defines the desired state of SupportBundle
2924
type SupportBundleSpec struct {
3025
AfterCollection []*AfterCollection `json:"afterCollection,omitempty" yaml:"afterCollection,omitempty"`
@@ -33,7 +28,8 @@ type SupportBundleSpec struct {
3328
Analyzers []*Analyze `json:"analyzers,omitempty" yaml:"analyzers,omitempty"`
3429
HostAnalyzers []*HostAnalyze `json:"hostAnalyzers,omitempty" yaml:"hostAnalyzers,omitempty"`
3530
// URI optionally defines a location which is the source of this spec to allow updating of the spec at runtime
36-
Uri string `json:"uri,omitempty" yaml:"uri,omitempty"`
31+
Uri string `json:"uri,omitempty" yaml:"uri,omitempty"`
32+
RunHostCollectorsInPod bool `json:"runHostCollectorsInPod,omitempty" yaml:"runHostCollectorsInPod,omitempty"`
3733
}
3834

3935
// SupportBundleStatus defines the observed state of SupportBundle
@@ -48,8 +44,8 @@ type SupportBundleStatus struct {
4844
// SupportBundle is the Schema for the SupportBundles API
4945
// +k8s:openapi-gen=true
5046
type SupportBundle struct {
51-
metav1.TypeMeta `json:",inline" yaml:",inline"`
52-
Metadata SupportBundleMetadata `json:"metadata,omitempty" yaml:"metadata,omitempty"`
47+
metav1.TypeMeta `json:",inline" yaml:",inline"`
48+
metav1.ObjectMeta `json:"metadata,omitempty" yaml:"metadata,omitempty"`
5349

5450
Spec SupportBundleSpec `json:"spec,omitempty" yaml:"spec,omitempty"`
5551
Status SupportBundleStatus `json:"status,omitempty"`

pkg/apis/troubleshoot/v1beta2/zz_generated.deepcopy.go

Lines changed: 1 addition & 17 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

pkg/client/troubleshootclientset/typed/troubleshoot/v1beta2/fake/fake_supportbundle.go

Lines changed: 1 addition & 1 deletion
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

pkg/client/troubleshootclientset/typed/troubleshoot/v1beta2/supportbundle.go

Lines changed: 2 additions & 2 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

pkg/collect/cluster_resources_test.go

Lines changed: 3 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -481,11 +481,9 @@ func TestCollectClusterResources_CustomResource(t *testing.T) {
481481

482482
// Create a CR
483483
sbObject := troubleshootv1beta2.SupportBundle{
484-
Metadata: troubleshootv1beta2.SupportBundleMetadata{
485-
ObjectMeta: metav1.ObjectMeta{
486-
Name: "supportbundle",
487-
Namespace: "default",
488-
},
484+
ObjectMeta: metav1.ObjectMeta{
485+
Name: "supportbundle",
486+
Namespace: "default",
489487
},
490488
TypeMeta: metav1.TypeMeta{
491489
Kind: "SupportBundle",

pkg/collect/host_collector.go

Lines changed: 158 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,14 +1,43 @@
11
package collect
22

33
import (
4+
"bytes"
5+
"context"
6+
"encoding/json"
7+
"fmt"
8+
"os"
9+
"path/filepath"
10+
"time"
11+
12+
"github.com/pkg/errors"
413
troubleshootv1beta2 "github.com/replicatedhq/troubleshoot/pkg/apis/troubleshoot/v1beta2"
14+
"github.com/replicatedhq/troubleshoot/pkg/constants"
15+
"golang.org/x/sync/errgroup"
16+
corev1 "k8s.io/api/core/v1"
17+
"k8s.io/apimachinery/pkg/runtime"
18+
"k8s.io/apiserver/pkg/storage/names"
19+
"k8s.io/client-go/kubernetes"
20+
"k8s.io/client-go/rest"
521
)
622

723
type HostCollector interface {
824
Title() string
925
IsExcluded() (bool, error)
1026
Collect(progressChan chan<- interface{}) (map[string][]byte, error)
11-
RemoteCollect(progressChan chan<- interface{}) (map[string][]byte, error) // RemoteCollect is used to priviledge pods to collect data from different nodes
27+
}
28+
29+
type RemoteCollectParams struct {
30+
ProgressChan chan<- interface{}
31+
HostCollector *troubleshootv1beta2.HostCollect
32+
BundlePath string
33+
ClientConfig *rest.Config // specify actual type
34+
Image string
35+
PullPolicy string // specify actual type if needed
36+
Timeout time.Duration // specify duration type if needed
37+
LabelSelector string
38+
NamePrefix string
39+
Namespace string
40+
Title string
1241
}
1342

1443
func GetHostCollector(collector *troubleshootv1beta2.HostCollect, bundlePath string) (HostCollector, bool) {
@@ -81,3 +110,131 @@ func hostCollectorTitleOrDefault(meta troubleshootv1beta2.HostCollectorMeta, def
81110
}
82111
return defaultTitle
83112
}
113+
114+
func RemoteHostCollect(ctx context.Context, params RemoteCollectParams) (map[string][]byte, error) {
115+
scheme := runtime.NewScheme()
116+
if err := corev1.AddToScheme(scheme); err != nil {
117+
return nil, errors.Wrap(err, "failed to add runtime scheme")
118+
}
119+
120+
client, err := kubernetes.NewForConfig(params.ClientConfig)
121+
if err != nil {
122+
return nil, err
123+
}
124+
125+
runner := &podRunner{
126+
client: client,
127+
scheme: scheme,
128+
image: params.Image,
129+
pullPolicy: params.PullPolicy,
130+
waitInterval: remoteCollectorDefaultInterval,
131+
}
132+
133+
// Get all the nodes where we should run.
134+
nodes, err := listNodesNamesInSelector(ctx, client, params.LabelSelector)
135+
if err != nil {
136+
return nil, errors.Wrap(err, "failed to get the list of nodes matching a nodeSelector")
137+
}
138+
139+
if params.NamePrefix == "" {
140+
params.NamePrefix = remoteCollectorNamePrefix
141+
}
142+
143+
result, err := runRemote(ctx, runner, nodes, params.HostCollector, names.SimpleNameGenerator, params.NamePrefix, params.Namespace)
144+
if err != nil {
145+
return nil, errors.Wrap(err, "failed to run collector remotely")
146+
}
147+
148+
allCollectedData := mapCollectorResultToOutput(result, params)
149+
output := NewResult()
150+
151+
// save the first result we find in the node and save it
152+
for node, result := range allCollectedData {
153+
var nodeResult map[string]string
154+
if err := json.Unmarshal(result, &nodeResult); err != nil {
155+
return nil, errors.Wrap(err, "failed to marshal node results")
156+
}
157+
158+
for file, collectorResult := range nodeResult {
159+
directory := filepath.Dir(file)
160+
fileName := filepath.Base(file)
161+
// expected file name for remote collectors will be the normal path separated by / and the node name
162+
output.SaveResult(params.BundlePath, fmt.Sprintf("%s/%s/%s", directory, node, fileName), bytes.NewBufferString(collectorResult))
163+
}
164+
}
165+
166+
// check if NODE_LIST_FILE exists
167+
_, err = os.Stat(constants.NODE_LIST_FILE)
168+
// if it not exists, save the nodes list
169+
if err != nil {
170+
nodesBytes, err := json.MarshalIndent(HostOSInfoNodes{Nodes: nodes}, "", " ")
171+
if err != nil {
172+
return nil, errors.Wrap(err, "failed to marshal host os info nodes")
173+
}
174+
output.SaveResult(params.BundlePath, constants.NODE_LIST_FILE, bytes.NewBuffer(nodesBytes))
175+
}
176+
return output, nil
177+
}
178+
179+
func runRemote(ctx context.Context, runner runner, nodes []string, collector *troubleshootv1beta2.HostCollect, nameGenerator names.NameGenerator, namePrefix string, namespace string) (map[string][]byte, error) {
180+
g, ctx := errgroup.WithContext(ctx)
181+
results := make(chan map[string][]byte, len(nodes))
182+
183+
for _, node := range nodes {
184+
node := node
185+
g.Go(func() error {
186+
// May need to evaluate error and log warning. Otherwise any error
187+
// here will cancel the context of other goroutines and no results
188+
// will be returned.
189+
return runner.run(ctx, collector, namespace, nameGenerator.GenerateName(namePrefix+"-"), node, results)
190+
})
191+
}
192+
193+
// Wait for all collectors to complete or return the first error.
194+
if err := g.Wait(); err != nil {
195+
return nil, errors.Wrap(err, "failed remote collection")
196+
}
197+
close(results)
198+
199+
output := make(map[string][]byte)
200+
for result := range results {
201+
r := result
202+
for k, v := range r {
203+
output[k] = v
204+
}
205+
}
206+
207+
return output, nil
208+
}
209+
210+
func mapCollectorResultToOutput(result map[string][]byte, params RemoteCollectParams) map[string][]byte {
211+
allCollectedData := make(map[string][]byte)
212+
213+
for k, v := range result {
214+
if curBytes, ok := allCollectedData[k]; ok {
215+
var curResults map[string]string
216+
if err := json.Unmarshal(curBytes, &curResults); err != nil {
217+
params.ProgressChan <- errors.Errorf("failed to read existing results for collector %s: %v\n", params.Title, err)
218+
continue
219+
}
220+
var newResults map[string]string
221+
if err := json.Unmarshal(v, &newResults); err != nil {
222+
params.ProgressChan <- errors.Errorf("failed to read new results for collector %s: %v\n", params.Title, err)
223+
continue
224+
}
225+
for file, data := range newResults {
226+
curResults[file] = data
227+
}
228+
combinedResults, err := json.Marshal(curResults)
229+
if err != nil {
230+
params.ProgressChan <- errors.Errorf("failed to combine results for collector %s: %v\n", params.Title, err)
231+
continue
232+
}
233+
allCollectedData[k] = combinedResults
234+
} else {
235+
allCollectedData[k] = v
236+
}
237+
238+
}
239+
return allCollectedData
240+
}

0 commit comments

Comments
 (0)