Skip to content
This repository was archived by the owner on Apr 24, 2023. It is now read-only.

Commit

Permalink
Debug log for large overhead containers (#17)
Browse files Browse the repository at this point in the history
* log if high overhead pod detected

* log more

* cpu in milli

* debug log

* format

* reuse var

* better msg

* log param

* move cpu + mem outside

* rm line

* move vars to top
  • Loading branch information
adambalogh authored and onursatici committed Mar 21, 2019
1 parent 269dab5 commit aa19903
Showing 1 changed file with 17 additions and 3 deletions.
20 changes: 17 additions & 3 deletions internal/extender/overhead.go
Original file line number Diff line number Diff line change
Expand Up @@ -25,10 +25,16 @@ import (
"github.com/palantir/witchcraft-go-logging/wlog/svclog/svc1log"
"github.com/palantir/witchcraft-go-logging/wlog/wapp"
"k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/resource"
"k8s.io/apimachinery/pkg/labels"
corelisters "k8s.io/client-go/listers/core/v1"
)

var (
oneCPU = resource.NewMilliQuantity(1000, resource.DecimalSI)
oneGiB = resource.NewQuantity(1*1024*1024*1024, resource.BinarySI)
)

// OverheadComputer computes non spark scheduler managed pods total resources periodically
type OverheadComputer struct {
podLister corelisters.PodLister
Expand Down Expand Up @@ -120,7 +126,7 @@ func (o *OverheadComputer) compute(ctx context.Context) {
if _, ok := currentOverhead[p.Spec.NodeName]; !ok {
currentOverhead[p.Spec.NodeName] = resources.Zero()
}
currentOverhead[p.Spec.NodeName].Add(podToResources(p))
currentOverhead[p.Spec.NodeName].Add(podToResources(ctx, p))
}
overhead := Overhead{}
for instanceGroup, nodeGroupResources := range rawOverhead {
Expand All @@ -146,10 +152,18 @@ func (o *OverheadComputer) compute(ctx context.Context) {
o.overheadLock.Unlock()
}

func podToResources(pod *v1.Pod) *resources.Resources {
func podToResources(ctx context.Context, pod *v1.Pod) *resources.Resources {
res := resources.Zero()
for _, c := range pod.Spec.Containers {
res.AddFromResourceList(c.Resources.Requests)
resourceRequests := c.Resources.Requests
if resourceRequests.Cpu().AsDec().Cmp(oneCPU.AsDec()) > 0 || resourceRequests.Memory().AsDec().Cmp(oneGiB.AsDec()) > 0 {
svc1log.FromContext(ctx).Debug("Container with no resource reservation has high resource requests",
svc1log.SafeParam("podName", pod.Name),
svc1log.SafeParam("nodeName", pod.Spec.NodeName),
svc1log.SafeParam("CPU", resourceRequests.Cpu()),
svc1log.SafeParam("Memory", resourceRequests.Memory()))
}
res.AddFromResourceList(resourceRequests)
}
return res
}
Expand Down

0 comments on commit aa19903

Please sign in to comment.