1
2
3
4
5
6
7
8
9
10
11
12
13
14
15 package driver
16
17 import (
18 "bytes"
19 "fmt"
20 "io"
21 "net/http"
22 "net/url"
23 "os"
24 "os/exec"
25 "path/filepath"
26 "runtime"
27 "strconv"
28 "strings"
29 "sync"
30 "time"
31
32 "github.com/google/pprof/internal/measurement"
33 "github.com/google/pprof/internal/plugin"
34 "github.com/google/pprof/profile"
35 )
36
37
38
39
40
41 func fetchProfiles(s *source, o *plugin.Options) (*profile.Profile, error) {
42 sources := make([]profileSource, 0, len(s.Sources))
43 for _, src := range s.Sources {
44 sources = append(sources, profileSource{
45 addr: src,
46 source: s,
47 })
48 }
49
50 bases := make([]profileSource, 0, len(s.Base))
51 for _, src := range s.Base {
52 bases = append(bases, profileSource{
53 addr: src,
54 source: s,
55 })
56 }
57
58 p, pbase, m, mbase, save, err := grabSourcesAndBases(sources, bases, o.Fetch, o.Obj, o.UI, o.HTTPTransport)
59 if err != nil {
60 return nil, err
61 }
62
63 if pbase != nil {
64 if s.DiffBase {
65 pbase.SetLabel("pprof::base", []string{"true"})
66 }
67 if s.Normalize {
68 err := p.Normalize(pbase)
69 if err != nil {
70 return nil, err
71 }
72 }
73 pbase.Scale(-1)
74 p, m, err = combineProfiles([]*profile.Profile{p, pbase}, []plugin.MappingSources{m, mbase})
75 if err != nil {
76 return nil, err
77 }
78 }
79
80
81 if err := o.Sym.Symbolize(s.Symbolize, m, p); err != nil {
82 return nil, err
83 }
84 p.RemoveUninteresting()
85 unsourceMappings(p)
86
87 if s.Comment != "" {
88 p.Comments = append(p.Comments, s.Comment)
89 }
90
91
92 if save {
93 dir, err := setTmpDir(o.UI)
94 if err != nil {
95 return nil, err
96 }
97
98 prefix := "pprof."
99 if len(p.Mapping) > 0 && p.Mapping[0].File != "" {
100 prefix += filepath.Base(p.Mapping[0].File) + "."
101 }
102 for _, s := range p.SampleType {
103 prefix += s.Type + "."
104 }
105
106 tempFile, err := newTempFile(dir, prefix, ".pb.gz")
107 if err == nil {
108 if err = p.Write(tempFile); err == nil {
109 o.UI.PrintErr("Saved profile in ", tempFile.Name())
110 }
111 }
112 if err != nil {
113 o.UI.PrintErr("Could not save profile: ", err)
114 }
115 }
116
117 if err := p.CheckValid(); err != nil {
118 return nil, err
119 }
120
121 return p, nil
122 }
123
124 func grabSourcesAndBases(sources, bases []profileSource, fetch plugin.Fetcher, obj plugin.ObjTool, ui plugin.UI, tr http.RoundTripper) (*profile.Profile, *profile.Profile, plugin.MappingSources, plugin.MappingSources, bool, error) {
125 wg := sync.WaitGroup{}
126 wg.Add(2)
127 var psrc, pbase *profile.Profile
128 var msrc, mbase plugin.MappingSources
129 var savesrc, savebase bool
130 var errsrc, errbase error
131 var countsrc, countbase int
132 go func() {
133 defer wg.Done()
134 psrc, msrc, savesrc, countsrc, errsrc = chunkedGrab(sources, fetch, obj, ui, tr)
135 }()
136 go func() {
137 defer wg.Done()
138 pbase, mbase, savebase, countbase, errbase = chunkedGrab(bases, fetch, obj, ui, tr)
139 }()
140 wg.Wait()
141 save := savesrc || savebase
142
143 if errsrc != nil {
144 return nil, nil, nil, nil, false, fmt.Errorf("problem fetching source profiles: %v", errsrc)
145 }
146 if errbase != nil {
147 return nil, nil, nil, nil, false, fmt.Errorf("problem fetching base profiles: %v,", errbase)
148 }
149 if countsrc == 0 {
150 return nil, nil, nil, nil, false, fmt.Errorf("failed to fetch any source profiles")
151 }
152 if countbase == 0 && len(bases) > 0 {
153 return nil, nil, nil, nil, false, fmt.Errorf("failed to fetch any base profiles")
154 }
155 if want, got := len(sources), countsrc; want != got {
156 ui.PrintErr(fmt.Sprintf("Fetched %d source profiles out of %d", got, want))
157 }
158 if want, got := len(bases), countbase; want != got {
159 ui.PrintErr(fmt.Sprintf("Fetched %d base profiles out of %d", got, want))
160 }
161
162 return psrc, pbase, msrc, mbase, save, nil
163 }
164
165
166
167
168 func chunkedGrab(sources []profileSource, fetch plugin.Fetcher, obj plugin.ObjTool, ui plugin.UI, tr http.RoundTripper) (*profile.Profile, plugin.MappingSources, bool, int, error) {
169 const chunkSize = 128
170
171 var p *profile.Profile
172 var msrc plugin.MappingSources
173 var save bool
174 var count int
175
176 for start := 0; start < len(sources); start += chunkSize {
177 end := start + chunkSize
178 if end > len(sources) {
179 end = len(sources)
180 }
181 chunkP, chunkMsrc, chunkSave, chunkCount, chunkErr := concurrentGrab(sources[start:end], fetch, obj, ui, tr)
182 switch {
183 case chunkErr != nil:
184 return nil, nil, false, 0, chunkErr
185 case chunkP == nil:
186 continue
187 case p == nil:
188 p, msrc, save, count = chunkP, chunkMsrc, chunkSave, chunkCount
189 default:
190 p, msrc, chunkErr = combineProfiles([]*profile.Profile{p, chunkP}, []plugin.MappingSources{msrc, chunkMsrc})
191 if chunkErr != nil {
192 return nil, nil, false, 0, chunkErr
193 }
194 if chunkSave {
195 save = true
196 }
197 count += chunkCount
198 }
199 }
200
201 return p, msrc, save, count, nil
202 }
203
204
205 func concurrentGrab(sources []profileSource, fetch plugin.Fetcher, obj plugin.ObjTool, ui plugin.UI, tr http.RoundTripper) (*profile.Profile, plugin.MappingSources, bool, int, error) {
206 wg := sync.WaitGroup{}
207 wg.Add(len(sources))
208 for i := range sources {
209 go func(s *profileSource) {
210 defer wg.Done()
211 s.p, s.msrc, s.remote, s.err = grabProfile(s.source, s.addr, fetch, obj, ui, tr)
212 }(&sources[i])
213 }
214 wg.Wait()
215
216 var save bool
217 profiles := make([]*profile.Profile, 0, len(sources))
218 msrcs := make([]plugin.MappingSources, 0, len(sources))
219 for i := range sources {
220 s := &sources[i]
221 if err := s.err; err != nil {
222 ui.PrintErr(s.addr + ": " + err.Error())
223 continue
224 }
225 save = save || s.remote
226 profiles = append(profiles, s.p)
227 msrcs = append(msrcs, s.msrc)
228 *s = profileSource{}
229 }
230
231 if len(profiles) == 0 {
232 return nil, nil, false, 0, nil
233 }
234
235 p, msrc, err := combineProfiles(profiles, msrcs)
236 if err != nil {
237 return nil, nil, false, 0, err
238 }
239 return p, msrc, save, len(profiles), nil
240 }
241
242 func combineProfiles(profiles []*profile.Profile, msrcs []plugin.MappingSources) (*profile.Profile, plugin.MappingSources, error) {
243
244
245
246
247
248 if err := profile.CompatibilizeSampleTypes(profiles); err != nil {
249 return nil, nil, err
250 }
251 if err := measurement.ScaleProfiles(profiles); err != nil {
252 return nil, nil, err
253 }
254
255
256 if len(profiles) == 1 && len(msrcs) == 1 {
257 return profiles[0], msrcs[0], nil
258 }
259
260 p, err := profile.Merge(profiles)
261 if err != nil {
262 return nil, nil, err
263 }
264
265
266 msrc := make(plugin.MappingSources)
267 for _, ms := range msrcs {
268 for m, s := range ms {
269 msrc[m] = append(msrc[m], s...)
270 }
271 }
272 return p, msrc, nil
273 }
274
275 type profileSource struct {
276 addr string
277 source *source
278
279 p *profile.Profile
280 msrc plugin.MappingSources
281 remote bool
282 err error
283 }
284
285 func homeEnv() string {
286 switch runtime.GOOS {
287 case "windows":
288 return "USERPROFILE"
289 case "plan9":
290 return "home"
291 default:
292 return "HOME"
293 }
294 }
295
296
297
298
299 func setTmpDir(ui plugin.UI) (string, error) {
300 var dirs []string
301 if profileDir := os.Getenv("PPROF_TMPDIR"); profileDir != "" {
302 dirs = append(dirs, profileDir)
303 }
304 if homeDir := os.Getenv(homeEnv()); homeDir != "" {
305 dirs = append(dirs, filepath.Join(homeDir, "pprof"))
306 }
307 dirs = append(dirs, os.TempDir())
308 for _, tmpDir := range dirs {
309 if err := os.MkdirAll(tmpDir, 0755); err != nil {
310 ui.PrintErr("Could not use temp dir ", tmpDir, ": ", err.Error())
311 continue
312 }
313 return tmpDir, nil
314 }
315 return "", fmt.Errorf("failed to identify temp dir")
316 }
317
318 const testSourceAddress = "pproftest.local"
319
320
321
322
323 func grabProfile(s *source, source string, fetcher plugin.Fetcher, obj plugin.ObjTool, ui plugin.UI, tr http.RoundTripper) (p *profile.Profile, msrc plugin.MappingSources, remote bool, err error) {
324 var src string
325 duration, timeout := time.Duration(s.Seconds)*time.Second, time.Duration(s.Timeout)*time.Second
326 if fetcher != nil {
327 p, src, err = fetcher.Fetch(source, duration, timeout)
328 if err != nil {
329 return
330 }
331 }
332 if err != nil || p == nil {
333
334 p, src, err = fetch(source, duration, timeout, ui, tr)
335 if err != nil {
336 return
337 }
338 }
339
340 if err = p.CheckValid(); err != nil {
341 return
342 }
343
344
345 locateBinaries(p, s, obj, ui)
346
347
348 if src != "" {
349 msrc = collectMappingSources(p, src)
350 remote = true
351 if strings.HasPrefix(src, "http://"+testSourceAddress) {
352
353
354 remote = false
355 }
356 }
357 return
358 }
359
360
361 func collectMappingSources(p *profile.Profile, source string) plugin.MappingSources {
362 ms := plugin.MappingSources{}
363 for _, m := range p.Mapping {
364 src := struct {
365 Source string
366 Start uint64
367 }{
368 source, m.Start,
369 }
370 key := m.BuildID
371 if key == "" {
372 key = m.File
373 }
374 if key == "" {
375
376
377
378
379
380 m.File = source
381 key = source
382 }
383 ms[key] = append(ms[key], src)
384 }
385 return ms
386 }
387
388
389
390 func unsourceMappings(p *profile.Profile) {
391 for _, m := range p.Mapping {
392 if m.BuildID == "" && filepath.VolumeName(m.File) == "" {
393 if u, err := url.Parse(m.File); err == nil && u.IsAbs() {
394 m.File = ""
395 }
396 }
397 }
398 }
399
400
401
402 func locateBinaries(p *profile.Profile, s *source, obj plugin.ObjTool, ui plugin.UI) {
403
404 searchPath := os.Getenv("PPROF_BINARY_PATH")
405 if searchPath == "" {
406
407 searchPath = filepath.Join(os.Getenv(homeEnv()), "pprof", "binaries")
408 }
409 mapping:
410 for _, m := range p.Mapping {
411 var noVolumeFile string
412 var baseName string
413 var dirName string
414 if m.File != "" {
415 noVolumeFile = strings.TrimPrefix(m.File, filepath.VolumeName(m.File))
416 baseName = filepath.Base(m.File)
417 dirName = filepath.Dir(noVolumeFile)
418 }
419
420 for _, path := range filepath.SplitList(searchPath) {
421 var fileNames []string
422 if m.BuildID != "" {
423 fileNames = []string{filepath.Join(path, m.BuildID, baseName)}
424 if matches, err := filepath.Glob(filepath.Join(path, m.BuildID, "*")); err == nil {
425 fileNames = append(fileNames, matches...)
426 }
427 fileNames = append(fileNames, filepath.Join(path, noVolumeFile, m.BuildID))
428
429
430
431 fileNames = append(fileNames, filepath.Join(path, m.BuildID[:2], m.BuildID[2:]+".debug"))
432 }
433 if m.File != "" {
434
435
436 fileNames = append(fileNames, filepath.Join(path, baseName))
437 fileNames = append(fileNames, filepath.Join(path, noVolumeFile))
438
439
440 fileNames = append(fileNames, filepath.Join(path, noVolumeFile+".debug"))
441 fileNames = append(fileNames, filepath.Join(path, dirName, ".debug", baseName+".debug"))
442 fileNames = append(fileNames, filepath.Join(path, "usr", "lib", "debug", dirName, baseName+".debug"))
443 }
444 for _, name := range fileNames {
445 if f, err := obj.Open(name, m.Start, m.Limit, m.Offset, m.KernelRelocationSymbol); err == nil {
446 defer f.Close()
447 fileBuildID := f.BuildID()
448 if m.BuildID != "" && m.BuildID != fileBuildID {
449 ui.PrintErr("Ignoring local file " + name + ": build-id mismatch (" + m.BuildID + " != " + fileBuildID + ")")
450 } else {
451
452
453 m.File = name
454 continue mapping
455 }
456 }
457 }
458 }
459 }
460 if len(p.Mapping) == 0 {
461
462
463
464
465 m := &profile.Mapping{ID: 1}
466 p.Mapping = []*profile.Mapping{m}
467 for _, l := range p.Location {
468 l.Mapping = m
469 }
470 }
471
472
473 if execName, buildID := s.ExecName, s.BuildID; execName != "" || buildID != "" {
474 m := p.Mapping[0]
475 if execName != "" {
476
477
478 m.File = execName
479 }
480
481
482
483 if buildID != "" && m.BuildID == "" {
484 m.BuildID = buildID
485 }
486 }
487 }
488
489
490
491
492 func fetch(source string, duration, timeout time.Duration, ui plugin.UI, tr http.RoundTripper) (p *profile.Profile, src string, err error) {
493 var f io.ReadCloser
494
495 if sourceURL, timeout := adjustURL(source, duration, timeout); sourceURL != "" {
496 ui.Print("Fetching profile over HTTP from " + sourceURL)
497 if duration > 0 {
498 ui.Print(fmt.Sprintf("Please wait... (%v)", duration))
499 }
500 f, err = fetchURL(sourceURL, timeout, tr)
501 src = sourceURL
502 } else if isPerfFile(source) {
503 f, err = convertPerfData(source, ui)
504 } else {
505 f, err = os.Open(source)
506 }
507 if err == nil {
508 defer f.Close()
509 p, err = profile.Parse(f)
510 }
511 return
512 }
513
514
515 func fetchURL(source string, timeout time.Duration, tr http.RoundTripper) (io.ReadCloser, error) {
516 client := &http.Client{
517 Transport: tr,
518 Timeout: timeout + 5*time.Second,
519 }
520 resp, err := client.Get(source)
521 if err != nil {
522 return nil, fmt.Errorf("http fetch: %v", err)
523 }
524 if resp.StatusCode != http.StatusOK {
525 defer resp.Body.Close()
526 return nil, statusCodeError(resp)
527 }
528
529 return resp.Body, nil
530 }
531
532 func statusCodeError(resp *http.Response) error {
533 if resp.Header.Get("X-Go-Pprof") != "" && strings.Contains(resp.Header.Get("Content-Type"), "text/plain") {
534
535 if body, err := io.ReadAll(resp.Body); err == nil {
536 return fmt.Errorf("server response: %s - %s", resp.Status, body)
537 }
538 }
539 return fmt.Errorf("server response: %s", resp.Status)
540 }
541
542
543
544 func isPerfFile(path string) bool {
545 sourceFile, openErr := os.Open(path)
546 if openErr != nil {
547 return false
548 }
549 defer sourceFile.Close()
550
551
552
553 perfHeader := []byte("PERFILE2")
554 actualHeader := make([]byte, len(perfHeader))
555 if _, readErr := sourceFile.Read(actualHeader); readErr != nil {
556 return false
557 }
558 return bytes.Equal(actualHeader, perfHeader)
559 }
560
561
562
563
564 func convertPerfData(perfPath string, ui plugin.UI) (*os.File, error) {
565 ui.Print(fmt.Sprintf(
566 "Converting %s to a profile.proto... (May take a few minutes)",
567 perfPath))
568 profile, err := newTempFile(os.TempDir(), "pprof_", ".pb.gz")
569 if err != nil {
570 return nil, err
571 }
572 deferDeleteTempFile(profile.Name())
573 cmd := exec.Command("perf_to_profile", "-i", perfPath, "-o", profile.Name(), "-f")
574 cmd.Stdout, cmd.Stderr = os.Stdout, os.Stderr
575 if err := cmd.Run(); err != nil {
576 profile.Close()
577 return nil, fmt.Errorf("failed to convert perf.data file. Try github.com/google/perf_data_converter: %v", err)
578 }
579 return profile, nil
580 }
581
582
583
584
585 func adjustURL(source string, duration, timeout time.Duration) (string, time.Duration) {
586 u, err := url.Parse(source)
587 if err != nil || (u.Host == "" && u.Scheme != "" && u.Scheme != "file") {
588
589
590 u, err = url.Parse("http://" + source)
591 }
592 if err != nil || u.Host == "" {
593 return "", 0
594 }
595
596
597 values := u.Query()
598 if duration > 0 {
599 values.Set("seconds", fmt.Sprint(int(duration.Seconds())))
600 } else {
601 if urlSeconds := values.Get("seconds"); urlSeconds != "" {
602 if us, err := strconv.ParseInt(urlSeconds, 10, 32); err == nil {
603 duration = time.Duration(us) * time.Second
604 }
605 }
606 }
607 if timeout <= 0 {
608 if duration > 0 {
609 timeout = duration + duration/2
610 } else {
611 timeout = 60 * time.Second
612 }
613 }
614 u.RawQuery = values.Encode()
615 return u.String(), timeout
616 }
617
View as plain text