1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
|
#!/usr/bin/env python3
"""
This file is part of Mbed TLS (https://tls.mbed.org)
Copyright (c) 2018, Arm Limited, All Rights Reserved
Purpose
This script checks the current state of the source code for minor issues,
including incorrect file permissions, presence of tabs, non-Unix line endings,
trailing whitespace, presence of UTF-8 BOM, and TODO comments.
Note: requires python 3, must be run from Mbed TLS root.
"""
import os
import argparse
import logging
import codecs
import sys
class IssueTracker(object):
"""Base class for issue tracking. Issues should inherit from this and
overwrite either issue_with_line if they check the file line by line, or
overwrite check_file_for_issue if they check the file as a whole."""
def __init__(self):
self.heading = ""
self.files_exemptions = []
self.files_with_issues = {}
def should_check_file(self, filepath):
for files_exemption in self.files_exemptions:
if filepath.endswith(files_exemption):
return False
return True
def issue_with_line(self, line):
raise NotImplementedError
def check_file_for_issue(self, filepath):
with open(filepath, "rb") as f:
for i, line in enumerate(iter(f.readline, b"")):
self.check_file_line(filepath, line, i + 1)
def record_issue(self, filepath, line_number):
if filepath not in self.files_with_issues.keys():
self.files_with_issues[filepath] = []
self.files_with_issues[filepath].append(line_number)
def check_file_line(self, filepath, line, line_number):
if self.issue_with_line(line):
self.record_issue(filepath, line_number)
def output_file_issues(self, logger):
if self.files_with_issues.values():
logger.info(self.heading)
for filename, lines in sorted(self.files_with_issues.items()):
if lines:
logger.info("{}: {}".format(
filename, ", ".join(str(x) for x in lines)
))
else:
logger.info(filename)
logger.info("")
class PermissionIssueTracker(IssueTracker):
def __init__(self):
super().__init__()
self.heading = "Incorrect permissions:"
def check_file_for_issue(self, filepath):
if not (os.access(filepath, os.X_OK) ==
filepath.endswith((".sh", ".pl", ".py"))):
self.files_with_issues[filepath] = None
class EndOfFileNewlineIssueTracker(IssueTracker):
def __init__(self):
super().__init__()
self.heading = "Missing newline at end of file:"
def check_file_for_issue(self, filepath):
with open(filepath, "rb") as f:
if not f.read().endswith(b"\n"):
self.files_with_issues[filepath] = None
class Utf8BomIssueTracker(IssueTracker):
def __init__(self):
super().__init__()
self.heading = "UTF-8 BOM present:"
def check_file_for_issue(self, filepath):
with open(filepath, "rb") as f:
if f.read().startswith(codecs.BOM_UTF8):
self.files_with_issues[filepath] = None
class LineEndingIssueTracker(IssueTracker):
def __init__(self):
super().__init__()
self.heading = "Non Unix line endings:"
def issue_with_line(self, line):
return b"\r" in line
class TrailingWhitespaceIssueTracker(IssueTracker):
def __init__(self):
super().__init__()
self.heading = "Trailing whitespace:"
self.files_exemptions = [".md"]
def issue_with_line(self, line):
return line.rstrip(b"\r\n") != line.rstrip()
class TabIssueTracker(IssueTracker):
def __init__(self):
super().__init__()
self.heading = "Tabs present:"
self.files_exemptions = [
"Makefile", "generate_visualc_files.pl"
]
def issue_with_line(self, line):
return b"\t" in line
class MergeArtifactIssueTracker(IssueTracker):
def __init__(self):
super().__init__()
self.heading = "Merge artifact:"
def issue_with_line(self, filepath, line):
# Detect leftover git conflict markers.
if line.startswith(b'<<<<<<< ') or line.startswith(b'>>>>>>> '):
return True
if line.startswith(b'||||||| '): # from merge.conflictStyle=diff3
return True
if line.rstrip(b'\r\n') == b'=======' and \
not filepath.endswith('.md'):
return True
return False
def check_file_line(self, filepath, line, line_number):
if self.issue_with_line(filepath, line):
self.record_issue(filepath, line_number)
class TodoIssueTracker(IssueTracker):
def __init__(self):
super().__init__()
self.heading = "TODO present:"
self.files_exemptions = [
os.path.basename(__file__),
"benchmark.c",
"pull_request_template.md",
]
def issue_with_line(self, line):
return b"todo" in line.lower()
class IntegrityChecker(object):
def __init__(self, log_file):
self.check_repo_path()
self.logger = None
self.setup_logger(log_file)
self.files_to_check = (
".c", ".h", ".sh", ".pl", ".py", ".md", ".function", ".data",
"Makefile", "CMakeLists.txt", "ChangeLog"
)
self.excluded_directories = ['.git', 'mbed-os']
self.excluded_paths = list(map(os.path.normpath, [
'cov-int',
'examples',
]))
self.issues_to_check = [
PermissionIssueTracker(),
EndOfFileNewlineIssueTracker(),
Utf8BomIssueTracker(),
LineEndingIssueTracker(),
TrailingWhitespaceIssueTracker(),
TabIssueTracker(),
MergeArtifactIssueTracker(),
TodoIssueTracker(),
]
def check_repo_path(self):
if not all(os.path.isdir(d) for d in ["include", "library", "tests"]):
raise Exception("Must be run from Mbed TLS root")
def setup_logger(self, log_file, level=logging.INFO):
self.logger = logging.getLogger()
self.logger.setLevel(level)
if log_file:
handler = logging.FileHandler(log_file)
self.logger.addHandler(handler)
else:
console = logging.StreamHandler()
self.logger.addHandler(console)
def prune_branch(self, root, d):
if d in self.excluded_directories:
return True
if os.path.normpath(os.path.join(root, d)) in self.excluded_paths:
return True
return False
def check_files(self):
for root, dirs, files in os.walk("."):
dirs[:] = sorted(d for d in dirs if not self.prune_branch(root, d))
for filename in sorted(files):
filepath = os.path.join(root, filename)
if not filepath.endswith(self.files_to_check):
continue
for issue_to_check in self.issues_to_check:
if issue_to_check.should_check_file(filepath):
issue_to_check.check_file_for_issue(filepath)
def output_issues(self):
integrity_return_code = 0
for issue_to_check in self.issues_to_check:
if issue_to_check.files_with_issues:
integrity_return_code = 1
issue_to_check.output_file_issues(self.logger)
return integrity_return_code
def run_main():
parser = argparse.ArgumentParser(
description=(
"This script checks the current state of the source code for "
"minor issues, including incorrect file permissions, "
"presence of tabs, non-Unix line endings, trailing whitespace, "
"presence of UTF-8 BOM, and TODO comments. "
"Note: requires python 3, must be run from Mbed TLS root."
)
)
parser.add_argument(
"-l", "--log_file", type=str, help="path to optional output log",
)
check_args = parser.parse_args()
integrity_check = IntegrityChecker(check_args.log_file)
integrity_check.check_files()
return_code = integrity_check.output_issues()
sys.exit(return_code)
if __name__ == "__main__":
run_main()
|
