Skip to content

Commit b131405

Browse files
adriansradriansr
authored
Harden system/package parsing of deb size (#17188)
* Harden system/package parsing of deb size Deb packages report their installed size in a field, Installed-Size, which is an integer interpreted as KiB. Some unofficial packages are adding a unit at the end of this field: Installed-Size: 65M System tools dpkg/apt ignore everything after the number. Auditbeat is currently failing to parse the list of installed packages once this mistake is reached. This updates the dataset to: - Do not fail when installed size can't be parsed. - Understand prefixes k/K, m/M and G/b. Fixes #16661
1 parent 57e194b commit b131405

File tree

4 files changed

+296
-4
lines changed

4 files changed

+296
-4
lines changed

‎CHANGELOG.next.asciidoc

+1
Original file line numberDiff line numberDiff line change
@@ -75,6 +75,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
7575
*Auditbeat*
7676

7777
- system/socket: Fixed compatibility issue with kernel 5.x. {pull}15771[15771]
78+
- system/package: Fix parsing of Installed-Size field of DEB packages. {issue}16661[16661] {pull}17188[17188]
7879

7980
*Filebeat*
8081

‎x-pack/auditbeat/module/system/package/package.go

+39-4
Original file line numberDiff line numberDiff line change
@@ -462,7 +462,7 @@ func (ms *MetricSet) getPackages() (packages []*Package, err error) {
462462
if err == nil {
463463
foundPackageManager = true
464464

465-
dpkgPackages, err := listDebPackages()
465+
dpkgPackages, err := ms.listDebPackages()
466466
if err != nil {
467467
return nil, errors.Wrap(err, "error getting DEB packages")
468468
}
@@ -499,7 +499,7 @@ func (ms *MetricSet) getPackages() (packages []*Package, err error) {
499499
return packages, nil
500500
}
501501

502-
func listDebPackages() ([]*Package, error) {
502+
func (ms *MetricSet) listDebPackages() ([]*Package, error) {
503503
dpkgStatusFile := filepath.Join(dpkgPath, "status")
504504

505505
file, err := os.Open(dpkgStatusFile)
@@ -556,9 +556,14 @@ func listDebPackages() ([]*Package, error) {
556556
case "description":
557557
pkg.Summary = value
558558
case "installed-size":
559-
pkg.Size, err = strconv.ParseUint(value, 10, 64)
559+
pkg.Size, err = parseDpkgInstalledSize(value)
560560
if err != nil {
561-
return nil, errors.Wrapf(err, "error converting %s to int", value)
561+
// If installed size is invalid, log a warning but still
562+
// report the package with size=0.
563+
ms.log.Warnw("Failed parsing installed size",
564+
"package", pkg.Name,
565+
"Installed-Size", value,
566+
"Error", err)
562567
}
563568
case "homepage":
564569
pkg.URL = value
@@ -578,3 +583,33 @@ func listDebPackages() ([]*Package, error) {
578583

579584
return packages, nil
580585
}
586+
587+
func parseDpkgInstalledSize(value string) (size uint64, err error) {
588+
// Installed-Size is an integer (KiB).
589+
if size, err = strconv.ParseUint(value, 10, 64); err == nil {
590+
return size, err
591+
}
592+
593+
// Some rare third-party packages contain a unit at the end. This is ignored
594+
// by dpkg tools. Try to parse to return a value as close as possible
595+
// to what the package maintainer meant.
596+
end := len(value)
597+
for idx, chr := range value {
598+
if chr < '0' || chr > '9' {
599+
end = idx
600+
break
601+
}
602+
}
603+
multiplier := uint64(1)
604+
if end < len(value) {
605+
switch value[end] {
606+
case 'm', 'M':
607+
multiplier = 1024
608+
case 'g', 'G':
609+
multiplier = 1024 * 1024
610+
}
611+
}
612+
613+
size, err = strconv.ParseUint(value[:end], 10, 64)
614+
return size * multiplier, err
615+
}

‎x-pack/auditbeat/module/system/package/package_test.go

+66
Original file line numberDiff line numberDiff line change
@@ -71,12 +71,78 @@ func TestDpkg(t *testing.T) {
7171
if assert.Len(t, events, 1) {
7272
event := mbtest.StandardizeEvent(f, events[0], core.AddDatasetToEvent)
7373
checkFieldValue(t, event, "system.audit.package.name", "test")
74+
checkFieldValue(t, event, "system.audit.package.size", uint64(269))
7475
checkFieldValue(t, event, "system.audit.package.summary", "Test Package")
7576
checkFieldValue(t, event, "system.audit.package.url", "https://www.elastic.co/")
7677
checkFieldValue(t, event, "system.audit.package.version", "8.2.0-1ubuntu2~18.04")
7778
}
7879
}
7980

81+
func TestDpkgInstalledSize(t *testing.T) {
82+
expected := map[string]uint64{
83+
"libquadmath0": 269,
84+
"python-apt-common": 248,
85+
"libnpth0": 32,
86+
"bind9-host": 17,
87+
"libpam-runtime": 300,
88+
"libsepol1-dev": 1739 * 1024,
89+
"libisl19": 17 * 1024 * 1024,
90+
"netbase": 0,
91+
"python2.7-minimal": 0,
92+
}
93+
94+
logp.TestingSetup()
95+
96+
defer abtest.SetupDataDir(t)()
97+
98+
// Disable all except dpkg
99+
rpmPathOld := rpmPath
100+
dpkgPathOld := dpkgPath
101+
brewPathOld := homebrewCellarPath
102+
defer func() {
103+
rpmPath = rpmPathOld
104+
dpkgPath = dpkgPathOld
105+
homebrewCellarPath = brewPathOld
106+
}()
107+
rpmPath = "/does/not/exist"
108+
homebrewCellarPath = "/does/not/exist"
109+
110+
var err error
111+
dpkgPath, err = filepath.Abs("testdata/dpkg-size/")
112+
if err != nil {
113+
t.Fatal(err)
114+
}
115+
116+
f := mbtest.NewReportingMetricSetV2(t, getConfig())
117+
defer f.(*MetricSet).bucket.DeleteBucket()
118+
119+
events, errs := mbtest.ReportingFetchV2(f)
120+
if len(errs) > 0 {
121+
t.Fatalf("received error: %+v", errs[0])
122+
}
123+
124+
got := make(map[string]uint64, len(events))
125+
for _, ev := range events {
126+
event := mbtest.StandardizeEvent(f, ev, core.AddDatasetToEvent)
127+
name, err := event.GetValue("system.audit.package.name")
128+
if err != nil {
129+
t.Fatal(err)
130+
}
131+
size, err := event.GetValue("system.audit.package.size")
132+
if err != nil {
133+
size = uint64(0)
134+
}
135+
if !assert.IsType(t, "", name) {
136+
t.Fatal("string expected")
137+
}
138+
if !assert.IsType(t, uint64(0), size) {
139+
t.Fatal("uint64 expected")
140+
}
141+
got[name.(string)] = size.(uint64)
142+
}
143+
assert.Equal(t, expected, got)
144+
}
145+
80146
func getConfig() map[string]interface{} {
81147
return map[string]interface{}{
82148
"module": "system",

‎x-pack/auditbeat/module/system/package/testdata/dpkg-size/status

+190
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,190 @@
1+
Package: libquadmath0
2+
Status: install ok installed
3+
Priority: optional
4+
Section: libs
5+
Installed-Size: 269B
6+
Maintainer: Ubuntu Core developers <ubuntu-devel-discuss@lists.ubuntu.com>
7+
Architecture: amd64
8+
Multi-Arch: same
9+
Source: gcc-8
10+
Version: 8.3.0-26ubuntu1~18.04
11+
Depends: gcc-8-base (= 8.3.0-26ubuntu1~18.04), libc6 (>= 2.23)
12+
Description: GCC Quad-Precision Math Library
13+
A library, which provides quad-precision mathematical functions on targets
14+
supporting the __float128 datatype. The library is used to provide on such
15+
targets the REAL(16) type in the GNU Fortran compiler.
16+
Homepage: http://gcc.gnu.org/
17+
Original-Maintainer: Debian GCC Maintainers <debian-gcc@lists.debian.org>
18+
19+
Package: python-apt-common
20+
Status: install ok installed
21+
Priority: optional
22+
Section: python
23+
Installed-Size: 248KiB
24+
Maintainer: Ubuntu Developers <ubuntu-devel-discuss@lists.ubuntu.com>
25+
Architecture: all
26+
Source: python-apt
27+
Version: 1.6.5ubuntu0.2
28+
Replaces: python-apt (<< 0.7.98+nmu1)
29+
Breaks: python-apt (<< 0.7.98+nmu1)
30+
Enhances: python-apt, python3-apt
31+
Description: Python interface to libapt-pkg (locales)
32+
The apt_pkg Python interface will provide full access to the internal
33+
libapt-pkg structures allowing Python programs to easily perform a
34+
variety of functions.
35+
.
36+
This package contains locales.
37+
Original-Maintainer: APT Development Team <deity@lists.debian.org>
38+
39+
Package: libnpth0
40+
Status: install ok installed
41+
Priority: optional
42+
Section: libs
43+
Installed-Size: 32trash
44+
Maintainer: Ubuntu Developers <ubuntu-devel-discuss@lists.ubuntu.com>
45+
Architecture: amd64
46+
Multi-Arch: same
47+
Source: npth
48+
Version: 1.5-3
49+
Depends: libc6 (>= 2.17)
50+
Description: replacement for GNU Pth using system threads
51+
nPth is a non-preemptive threads implementation using an API very
52+
similar to the one known from GNU Pth. It has been designed as a
53+
replacement of GNU Pth for non-ancient operating systems. In
54+
contrast to GNU Pth it is based on the system's standard threads
55+
implementation. Thus nPth allows the use of libraries which are not
56+
compatible to GNU Pth.
57+
Original-Maintainer: Eric Dorland <eric@debian.org>
58+
Homepage: https://www.gnupg.org/
59+
60+
Package: bind9-host
61+
Status: install ok installed
62+
Priority: standard
63+
Section: net
64+
Installed-Size: 17.4
65+
Maintainer: Ubuntu Developers <ubuntu-devel-discuss@lists.ubuntu.com>
66+
Architecture: amd64
67+
Source: bind9
68+
Version: 1:9.11.3+dfsg-1ubuntu1.11
69+
Provides: host
70+
Depends: libbind9-160 (= 1:9.11.3+dfsg-1ubuntu1.11), libdns1100 (= 1:9.11.3+dfsg-1ubuntu1.11), libisc169 (= 1:9.11.3+dfsg-1ubuntu1.11), libisccfg160 (= 1:9.11.3+dfsg-1ubuntu1.11), liblwres160 (= 1:9.11.3+dfsg-1ubuntu1.11), libc6 (>= 2.4)
71+
Description: DNS lookup utility (deprecated)
72+
This package provides /usr/bin/host, a simple utility (bundled with the
73+
BIND 9.X sources) which can be used for converting domain names to IP
74+
addresses and the reverse.
75+
.
76+
This utility is deprecated, use dig or delv from the dnsutils package.
77+
Homepage: https://www.isc.org/downloads/bind/
78+
Original-Maintainer: Debian DNS Packaging <pkg-dns-devel@lists.alioth.debian.org>
79+
80+
Package: libpam-runtime
81+
Status: install ok installed
82+
Priority: required
83+
Section: admin
84+
Installed-Size: 300T
85+
Maintainer: Ubuntu Developers <ubuntu-devel-discuss@lists.ubuntu.com>
86+
Architecture: all
87+
Multi-Arch: foreign
88+
Source: pam
89+
Version: 1.1.8-3.6ubuntu2
90+
Replaces: libpam0g-dev, libpam0g-util
91+
Depends: debconf (>= 0.5) | debconf-2.0, debconf (>= 1.5.19) | cdebconf, libpam-modules (>= 1.0.1-6)
92+
Conflicts: libpam0g-util
93+
Conffiles:
94+
/etc/pam.conf 87fc76f18e98ee7d3848f6b81b3391e5
95+
/etc/pam.d/other 31aa7f2181889ffb00b87df4126d1701
96+
Description: Runtime support for the PAM library
97+
Contains configuration files and directories required for
98+
authentication to work on Debian systems. This package is required
99+
on almost all installations.
100+
Homepage: http://www.linux-pam.org/
101+
Original-Maintainer: Steve Langasek <vorlon@debian.org>
102+
103+
Package: libsepol1-dev
104+
Status: install ok installed
105+
Priority: optional
106+
Section: libdevel
107+
Installed-Size: 1739m
108+
Maintainer: Ubuntu Developers <ubuntu-devel-discuss@lists.ubuntu.com>
109+
Architecture: amd64
110+
Multi-Arch: same
111+
Source: libsepol
112+
Version: 2.7-1
113+
Provides: libsepol-dev
114+
Depends: libsepol1 (= 2.7-1)
115+
Conflicts: libsepol-dev
116+
Description: SELinux binary policy manipulation library and development files
117+
libsepol allows programs to easily modify SELinux binary policies. This
118+
means changing the default values for booleans, or reading the policy for
119+
analysis.
120+
.
121+
This package contains the headers and archives used for linking it into your
122+
programs.
123+
Original-Maintainer: Debian SELinux maintainers <selinux-devel@lists.alioth.debian.org>
124+
Homepage: http://userspace.selinuxproject.org/
125+
126+
Package: libisl19
127+
Status: install ok installed
128+
Priority: optional
129+
Section: libs
130+
Installed-Size: 17G
131+
Maintainer: Ubuntu Developers <ubuntu-devel-discuss@lists.ubuntu.com>
132+
Architecture: amd64
133+
Multi-Arch: same
134+
Source: isl
135+
Version: 0.19-1
136+
Replaces: libisl-dbg (<< 0.19)
137+
Depends: libc6 (>= 2.14), libgmp10
138+
Breaks: libisl-dbg (<< 0.19)
139+
Description: manipulating sets and relations of integer points bounded by linear constraints
140+
isl is a library for manipulating sets and relations of integer points
141+
bounded by linear constraints. Supported operations on sets include
142+
intersection, union, set difference, emptiness check, convex hull,
143+
(integer) affine hull, integer projection, and computing the lexicographic
144+
minimum using parametric integer programming. It also includes an ILP solver
145+
based on generalized basis reduction.
146+
.
147+
This package contains the runtime library.
148+
Original-Maintainer: Debian GCC Maintainers <debian-gcc@lists.debian.org>
149+
Homepage: http://isl.gforge.inria.fr/
150+
151+
Package: netbase
152+
Status: install ok installed
153+
Priority: important
154+
Section: admin
155+
Maintainer: Ubuntu Developers <ubuntu-devel-discuss@lists.ubuntu.com>
156+
Architecture: all
157+
Multi-Arch: foreign
158+
Version: 5.4
159+
Conffiles:
160+
/etc/protocols bb9c019d6524e913fd72441d58b68216
161+
/etc/rpc f0b6f6352bf886623adc04183120f83b
162+
/etc/services 567c100888518c1163b3462993de7d47
163+
Description: Basic TCP/IP networking system
164+
This package provides the necessary infrastructure for basic TCP/IP based
165+
networking.
166+
Original-Maintainer: Marco d'Itri <md@linux.it>
167+
168+
Package: python2.7-minimal
169+
Status: install ok installed
170+
Priority: optional
171+
Section: python
172+
Installed-Size: Who knows
173+
Maintainer: Ubuntu Developers <ubuntu-devel-discuss@lists.ubuntu.com>
174+
Architecture: amd64
175+
Multi-Arch: allowed
176+
Source: python2.7
177+
Version: 2.7.17-1~18.04
178+
Replaces: python2.7 (<< 2.7.8-7~)
179+
Depends: libpython2.7-minimal (= 2.7.17-1~18.04)
180+
Pre-Depends: libc6 (>= 2.15), zlib1g (>= 1:1.2.0)
181+
Recommends: python2.7
182+
Suggests: binfmt-support
183+
Conflicts: binfmt-support (<< 1.1.2)
184+
Description: Minimal subset of the Python language (version 2.7)
185+
This package contains the interpreter and some essential modules. It can
186+
be used in the boot process for some basic tasks.
187+
See /usr/share/doc/python2.7-minimal/README.Debian for a list of the modules
188+
contained in this package.
189+
Original-Maintainer: Matthias Klose <doko@debian.org>
190+

0 commit comments

Comments
 (0)