Skip to content

Commit 122d721

Browse files
lazarkovlazarkov
authored
Merge branch 'master' into feature/add-logic-message-cluster
2 parents 24719e1 + 22ffcf0 commit 122d721

File tree

2 files changed

+342
-0
lines changed

2 files changed

+342
-0
lines changed

‎.github/workflows/lint.yml

+160
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,166 @@ jobs:
4242
with:
4343
platform: linux
4444

45+
- name: Check for orphaned gn files
46+
if: always()
47+
# We should enforce that ALL new files are referenced in our build scripts.
48+
# Several things do not have a clear fix path:
49+
# - various platform implementations (including darwin-specific files as they
50+
# are not using GN)
51+
# - app/clusters (they are fetched dynamically - this should probably be fixed)
52+
#
53+
# All the rest of the exceptions should be driven down to 0: chip should fully
54+
# be defined in build rules.
55+
#
56+
# This check enforces that for any newly added file, it must be part of some
57+
# BUILD.gn file
58+
run: |
59+
./scripts/run_in_build_env.sh "./scripts/tools/not_known_to_gn.py \
60+
src \
61+
--skip-dir app/clusters \
62+
--skip-dir darwin \
63+
--skip-dir include \
64+
--skip-dir platform/Ameba \
65+
--skip-dir platform/android \
66+
--skip-dir platform/ASR \
67+
--skip-dir platform/Beken \
68+
--skip-dir platform/bouffalolab \
69+
--skip-dir platform/cc13xx_26xx \
70+
--skip-dir platform/cc32xx \
71+
--skip-dir platform/Darwin \
72+
--skip-dir platform/ESP32 \
73+
--skip-dir platform/fake \
74+
--skip-dir platform/FreeRTOS \
75+
--skip-dir platform/Infineon \
76+
--skip-dir platform/Linux \
77+
--skip-dir platform/mbed \
78+
--skip-dir platform/mt793x \
79+
--skip-dir platform/nxp \
80+
--skip-dir platform/OpenThread \
81+
--skip-dir platform/qpg \
82+
--skip-dir platform/silabs \
83+
--skip-dir platform/telink \
84+
--skip-dir platform/webos \
85+
--skip-dir platform/Zephyr \
86+
--skip-dir test_driver \
87+
--known-failure app/app-platform/ContentApp.cpp \
88+
--known-failure app/app-platform/ContentApp.h \
89+
--known-failure app/app-platform/ContentAppPlatform.cpp \
90+
--known-failure app/app-platform/ContentAppPlatform.h \
91+
--known-failure controller/ExamplePersistentStorage.cpp \
92+
--known-failure controller/ExamplePersistentStorage.h \
93+
--known-failure controller/java/GroupDeviceProxy.h \
94+
--known-failure controller/java/CHIPEventTLVValueDecoder.h \
95+
--known-failure controller/python/chip/credentials/cert.h \
96+
--known-failure controller/python/chip/server/Options.h \
97+
--known-failure controller/python/chip/crypto/p256keypair.h \
98+
--known-failure controller/python/chip/commissioning/PlaceholderOperationalCredentialsIssuer.h \
99+
--known-failure controller/python/chip/native/PyChipError.h \
100+
--known-failure app/AttributeAccessInterface.h \
101+
--known-failure app/AttributeAccessToken.h \
102+
--known-failure app/att-storage.h \
103+
--known-failure app/BufferedReadCallback.h \
104+
--known-failure app/CommandHandler.h \
105+
--known-failure app/CommandHandlerInterface.h \
106+
--known-failure app/CommandPathParams.h \
107+
--known-failure app/CommandPathRegistry.h \
108+
--known-failure app/CommandResponseSender.h \
109+
--known-failure app/CommandSender.h \
110+
--known-failure app/CommandSenderLegacyCallback.h \
111+
--known-failure app/CompatEnumNames.h \
112+
--known-failure app/ConcreteAttributePath.h \
113+
--known-failure app/ConcreteCommandPath.h \
114+
--known-failure app/data-model/ListLargeSystemExtensions.h \
115+
--known-failure app/EventHeader.h \
116+
--known-failure app/EventLoggingDelegate.h \
117+
--known-failure app/EventLogging.h \
118+
--known-failure app/EventLoggingTypes.h \
119+
--known-failure app/EventManagement.h \
120+
--known-failure app/InteractionModelHelper.h \
121+
--known-failure app/MessageDef/ArrayBuilder.h \
122+
--known-failure app/MessageDef/ArrayParser.h \
123+
--known-failure app/MessageDef/CommandDataIB.h \
124+
--known-failure app/MessageDef/CommandPathIB.h \
125+
--known-failure app/MessageDef/CommandStatusIB.h \
126+
--known-failure app/MessageDef/EventFilterIB.h \
127+
--known-failure app/MessageDef/EventFilterIBs.h \
128+
--known-failure app/MessageDef/InvokeRequestMessage.h \
129+
--known-failure app/MessageDef/InvokeRequests.h \
130+
--known-failure app/MessageDef/InvokeResponseIB.h \
131+
--known-failure app/MessageDef/InvokeResponseIBs.h \
132+
--known-failure app/MessageDef/InvokeResponseMessage.h \
133+
--known-failure app/MessageDef/ListBuilder.h \
134+
--known-failure app/MessageDef/ListParser.h \
135+
--known-failure app/MessageDef/StatusResponseMessage.h \
136+
--known-failure app/MessageDef/StructBuilder.h \
137+
--known-failure app/MessageDef/StructParser.h \
138+
--known-failure app/MessageDef/SubscribeRequestMessage.h \
139+
--known-failure app/MessageDef/SubscribeResponseMessage.h \
140+
--known-failure app/MessageDef/TimedRequestMessage.h \
141+
--known-failure app/MessageDef/WriteRequestMessage.h \
142+
--known-failure app/MessageDef/WriteResponseMessage.h \
143+
--known-failure app/ObjectList.h \
144+
--known-failure app/ReadClient.h \
145+
--known-failure app/ReadHandler.h \
146+
--known-failure app/ReadPrepareParams.h \
147+
--known-failure app/reporting/tests/MockReportScheduler.cpp \
148+
--known-failure app/reporting/tests/MockReportScheduler.h \
149+
--known-failure app/server/AppDelegate.h \
150+
--known-failure app/TestEventTriggerDelegate.h \
151+
--known-failure app/tests/integration/common.h \
152+
--known-failure app/tests/integration/MockEvents.h \
153+
--known-failure app/tests/suites/credentials/TestHarnessDACProvider.h \
154+
--known-failure app/tests/TestOperationalDeviceProxy.cpp \
155+
--known-failure app/util/af-enums.h \
156+
--known-failure app/util/af.h \
157+
--known-failure app/util/af-types.h \
158+
--known-failure app/util/attribute-metadata.h \
159+
--known-failure app/util/attribute-storage.cpp \
160+
--known-failure app/util/attribute-storage.h \
161+
--known-failure app/util/attribute-storage-null-handling.h \
162+
--known-failure app/util/attribute-table.cpp \
163+
--known-failure app/util/attribute-table.h \
164+
--known-failure app/util/binding-table.cpp \
165+
--known-failure app/util/binding-table.h \
166+
--known-failure app/util/common.h \
167+
--known-failure app/util/config.h \
168+
--known-failure app/util/DataModelHandler.cpp \
169+
--known-failure app/util/DataModelHandler.h \
170+
--known-failure app/util/ember-compatibility-functions.cpp \
171+
--known-failure app/util/endpoint-config-api.h \
172+
--known-failure app/util/endpoint-config-defines.h \
173+
--known-failure app/util/error-mapping.h \
174+
--known-failure app/util/generic-callbacks.h \
175+
--known-failure app/util/generic-callback-stubs.cpp \
176+
--known-failure app/util/im-client-callbacks.h \
177+
--known-failure app/util/MatterCallbacks.h \
178+
--known-failure app/util/message.cpp \
179+
--known-failure app/util/mock/Constants.h \
180+
--known-failure app/util/mock/Functions.h \
181+
--known-failure app/util/mock/MockNodeConfig.h \
182+
--known-failure app/util/odd-sized-integers.h \
183+
--known-failure app/util/types_stub.h \
184+
--known-failure app/util/util.cpp \
185+
--known-failure app/util/util.h \
186+
--known-failure app/WriteClient.h \
187+
--known-failure app/WriteHandler.h \
188+
--known-failure inet/tests/TestInetLayerCommon.hpp \
189+
--known-failure lib/core/CHIPVendorIdentifiers.hpp \
190+
--known-failure lib/dnssd/Constants.h \
191+
--known-failure lib/dnssd/minimal_mdns/core/FlatAllocatedQName.h \
192+
--known-failure lib/dnssd/minimal_mdns/core/HeapQName.h \
193+
--known-failure lib/dnssd/minimal_mdns/ListenIterator.h \
194+
--known-failure lib/dnssd/minimal_mdns/tests/CheckOnlyServer.h \
195+
--known-failure lib/dnssd/platform/DnssdBrowseDelegate.h \
196+
--known-failure lib/support/CHIPArgParser.hpp \
197+
--known-failure messaging/tests/echo/common.h \
198+
--known-failure platform/DeviceSafeQueue.cpp \
199+
--known-failure platform/DeviceSafeQueue.h \
200+
--known-failure platform/GLibTypeDeleter.h \
201+
--known-failure platform/SingletonConfigurationManager.cpp \
202+
--known-failure transport/retransmit/tests/TestCacheDriver.cpp \
203+
"
204+
45205
- name: Check for matter lint errors
46206
if: always()
47207
run: |

‎scripts/tools/not_known_to_gn.py

+182
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,182 @@
1+
#!/usr/bin/env python3
2+
#
3+
# Copyright (c) 2024 Project CHIP Authors
4+
#
5+
# Licensed under the Apache License, Version 2.0 (the "License");
6+
# you may not use this file except in compliance with the License.
7+
# You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing, software
12+
# distributed under the License is distributed on an "AS IS" BASIS,
13+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
# See the License for the specific language governing permissions and
15+
# limitations under the License.
16+
#
17+
"""
18+
Lists files specific files from a source tree and ensures
19+
they are covered by GN in some way.
20+
21+
'Covered' is very loosely and it just tries to see if the GN text
22+
contains that word without trying to validate if this is a
23+
comment or some actual 'source' element.
24+
25+
It is intended as a failsafe to not foget adding source files
26+
to gn.
27+
"""
28+
import logging
29+
import os
30+
import sys
31+
from pathlib import Path, PurePath
32+
from typing import Dict, Set
33+
34+
import click
35+
import coloredlogs
36+
37+
__LOG_LEVELS__ = {
38+
'debug': logging.DEBUG,
39+
'info': logging.INFO,
40+
'warn': logging.WARN,
41+
'fatal': logging.FATAL,
42+
}
43+
44+
45+
class OrphanChecker:
46+
def __init__(self):
47+
self.gn_data: Dict[str, str] = {}
48+
self.known_failures: Set[str] = set()
49+
self.fatal_failures = 0
50+
self.failures = 0
51+
self.found_failures: Set[str] = set()
52+
53+
def AppendGnData(self, gn: PurePath):
54+
"""Adds a GN file to the list of internally known GN data.
55+
56+
Will read the entire content of the GN file in memory for future reference.
57+
"""
58+
logging.debug(f'Adding GN {gn!s} for {gn.parent!s}')
59+
self.gn_data[str(gn.parent)] = gn.read_text('utf-8')
60+
61+
def AddKnownFailure(self, k: str):
62+
self.known_failures.add(k)
63+
64+
def _IsKnownFailure(self, path: str) -> bool:
65+
"""check if failing on the given path is a known/acceptable failure"""
66+
for k in self.known_failures:
67+
if path == k or path.endswith(os.path.sep + k):
68+
# mark some found failures to report if something is supposed
69+
# to be known but it is not
70+
self.found_failures.add(k)
71+
return True
72+
return False
73+
74+
def Check(self, top_dir: str, file: PurePath):
75+
"""
76+
Validates that the given path is somehow referenced in GN files in any
77+
of the parent sub-directories of the file.
78+
79+
`file` must be relative to `top_dir`. Top_dir is used to resolve relative
80+
paths in error reports and known failure checks.
81+
"""
82+
# Check logic:
83+
# - ensure the file name is included in some GN file inside this or
84+
# upper directory (although upper directory is not ideal)
85+
for p in file.parents:
86+
data = self.gn_data.get(str(p), None)
87+
if not data:
88+
continue
89+
90+
if file.name in data:
91+
logging.debug("%s found in BUILD.gn for %s", file, p)
92+
return
93+
94+
path = str(file.relative_to(top_dir))
95+
if not self._IsKnownFailure(path):
96+
logging.error("UNKNOWN to gn: %s", path)
97+
self.fatal_failures += 1
98+
else:
99+
logging.warning("UNKNOWN to gn: %s (known error)", path)
100+
101+
self.failures += 1
102+
103+
104+
@click.command()
105+
@click.option(
106+
'--log-level',
107+
default='INFO',
108+
type=click.Choice(list(__LOG_LEVELS__.keys()), case_sensitive=False),
109+
help='Determines the verbosity of script output',
110+
)
111+
@click.option(
112+
'--extensions',
113+
default=["cpp", "cc", "c", "h", "hpp"],
114+
type=str, multiple=True,
115+
help='What file extensions to consider',
116+
)
117+
@click.option(
118+
'--known-failure',
119+
type=str, multiple=True,
120+
help='What paths are known to fail',
121+
)
122+
@click.option(
123+
'--skip-dir',
124+
type=str,
125+
multiple=True,
126+
help='Skip a specific sub-directory from checks',
127+
)
128+
@click.argument('dirs',
129+
type=click.Path(exists=True, file_okay=False, resolve_path=True), nargs=-1)
130+
def main(log_level, extensions, dirs, known_failure, skip_dir):
131+
coloredlogs.install(level=__LOG_LEVELS__[log_level],
132+
fmt='%(asctime)s %(levelname)-7s %(message)s')
133+
134+
if not dirs:
135+
logging.error("Please provide at least one directory to scan")
136+
sys.exit(1)
137+
138+
if not extensions:
139+
logging.error("Need at least one extension")
140+
sys.exit(1)
141+
142+
checker = OrphanChecker()
143+
for k in known_failure:
144+
checker.AddKnownFailure(k)
145+
146+
# ensure all GN data is loaded
147+
for directory in dirs:
148+
for name in Path(directory).rglob("BUILD.gn"):
149+
checker.AppendGnData(name)
150+
151+
skip_dir = set(skip_dir)
152+
153+
# Go through all files and check for orphaned (if any)
154+
extensions = set(extensions)
155+
for directory in dirs:
156+
for path, dirnames, filenames in os.walk(directory):
157+
if any([s in path for s in skip_dir]):
158+
continue
159+
for f in filenames:
160+
full_path = Path(os.path.join(path, f))
161+
if not full_path.suffix or full_path.suffix[1:] not in extensions:
162+
continue
163+
checker.Check(directory, full_path)
164+
165+
if checker.failures:
166+
logging.warning("%d files not known to GN (%d fatal)", checker.failures, checker.fatal_failures)
167+
168+
if checker.known_failures != checker.found_failures:
169+
not_failing = checker.known_failures - checker.found_failures
170+
logging.warning("NOTE: %d failures are not found anymore:", len(not_failing))
171+
for name in not_failing:
172+
logging.warning(" - %s", name)
173+
# Assume this is fatal - remove some of the "known-failing" should be easy.
174+
# This forces scripts to always be correct and not accumulate bad input.
175+
sys.exit(1)
176+
177+
if checker.fatal_failures > 0:
178+
sys.exit(1)
179+
180+
181+
if __name__ == '__main__':
182+
main(auto_envvar_prefix='CHIP')

0 commit comments

Comments
 (0)