[Skiboot] [PATCH skiboot v2 1/2] external/fwts: Add parser to extract olog pattern definitions
Jeremy Kerr
jk at ozlabs.org
Thu Apr 28 10:41:53 AEST 2016
The fwts project has a facility to scan system logs for interesting
error messages, by matching on patterns in a JSON file. Recently, Deb
has added support for the OPAL msglog to fwts, called 'olog':
http://kernel.ubuntu.com/git/hwe/fwts.git/commit/?id=652b79b
However, we don't yet have any patterns for OPAL.
Rather than generate a separate set of patterns that may go stale, Anton
suggested that we may want to pull these directly from the OPAL source.
This change implements a parser to generate olog pattern definitions
from annotations in OPAL itself. For example, a check in the flash code
might look like:
if (!ffs) {
/**
* @fwts-label SystemFlashNoPartitionTable
* @fwts-advice OPAL Could not read a partition table on
* system flash. Since we've still booted the machine (which
* requires flash), check that we're registering the proper
* system flash device.
*/
prlog(PR_WARNING, "FLASH: attempted to register system flash "
"%s, wwhich has no partition info\n", name);
return;
}
By running generate-fwts-olog on the codebase, we get:
{
"olog_error_warning_patterns": [
{
"advice": "OPAL Could not read a partition table on system flash. Since we've still booted the machine (which requires flash), check that we're registering the proper system flash device.",
"compare_mode": "regex",
"label": "SystemFlashNoPartitionTable",
"level": "LOG_LEVEL_HIGH",
"pattern": "FLASH: attempted to register system flash .*, wwhich has no partition info"
}
]
}
- which is suitable for input to the fwts pattern definitions.
Signed-off-by: Jeremy Kerr <jk at ozlabs.org>
--
v2:
- fix log_level (not 'level') in output dict
- properly line-break argparse definition
---
external/fwts/generate-fwts-olog | 229 +++++++++++++++++++++++++++++++++++++++
1 file changed, 229 insertions(+)
create mode 100755 external/fwts/generate-fwts-olog
diff --git a/external/fwts/generate-fwts-olog b/external/fwts/generate-fwts-olog
new file mode 100755
index 0000000..691890e
--- /dev/null
+++ b/external/fwts/generate-fwts-olog
@@ -0,0 +1,229 @@
+#!/usr/bin/env python2
+#
+# Copyright 2016 Jeremy Kerr <jk at ozlabs.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os.path
+import re
+import sys
+import string
+import json
+import argparse
+from pyparsing import Regex, Literal, Word, Combine, OneOrMore, QuotedString, \
+ lineno
+
+json_params = {
+ 'indent': 1,
+ 'sort_keys': True,
+}
+
+def create_parser():
+ # Match a C-style comment starting with two *s
+ comment = Regex(r'/\*\*(?P<content>.*?)\*/', re.DOTALL)
+
+ # Match an @fwts-<tag> annotation (within the comment), plus the proceeding
+ # text
+ annotation = Regex(r'@fwts-(?P<tag>\w+)\W+(?P<text>.*?)(?=@fwts-|\Z)',
+ re.DOTALL)
+
+ # Match the following prlog() call
+ log_call = (Literal("prlog") +
+ Literal('(').suppress() +
+ Word(string.letters + string.digits + '_') +
+ Literal(',').suppress() +
+ Combine(OneOrMore(QuotedString('"')), adjacent=False) +
+ (Literal(')') | Literal(',')).suppress()
+ )
+
+ pattern = comment + log_call
+ pattern.setWhitespaceChars(string.whitespace + '\n')
+
+ def comment_action(tok):
+ patterns = {}
+ for result in annotation.scanString(tok['content']):
+ patterns.update(result[0][0])
+ return patterns
+
+ def annotation_action(tok):
+ return {
+ tok['tag']: cleanup_content(tok['text'])
+ }
+
+ comment.setParseAction(comment_action)
+ annotation.setParseAction(annotation_action)
+ pattern.parseWithTabs()
+
+ return pattern
+
+def find_sources(dirname):
+ sources = []
+
+ def is_source(fname):
+ return fname.endswith('.c')
+
+ def add_fn(s, dname, fnames):
+ s.extend([ os.path.join(dname, fname)
+ for fname in fnames if is_source(fname) ])
+
+ os.path.walk(dirname, add_fn, sources)
+ return sources
+
+def cleanup_content(content):
+ comment_prefix_re = re.compile(r'^\s*\*\s*', re.MULTILINE)
+ whitespace_re = re.compile(r'\s+')
+
+ content = comment_prefix_re.sub(' ', content)
+ content = whitespace_re.sub(' ', content)
+ return content.strip()
+
+def warn(loc, message):
+ print >>sys.stderr, 'WARNING:%s:%d: %s' % (loc[0], loc[1], message)
+
+def log_level_to_fwts(level):
+ level_map = {
+ 'PR_EMERG': 'LOG_LEVEL_CRITICAL',
+ 'PR_ALERT': 'LOG_LEVEL_CRITICAL',
+ 'PR_CRIT': 'LOG_LEVEL_CRITICAL',
+ 'PR_ERR': 'LOG_LEVEL_CRITICAL',
+ 'PR_WARNING': 'LOG_LEVEL_HIGH',
+ 'PR_NOTICE': 'LOG_LEVEL_MEDIUM',
+ 'PR_PRINTF': 'LOG_LEVEL_MEDIUM',
+ }
+ return level_map.get(level, 'LOG_LEVEL_LOW')
+
+def message_to_pattern(loc, msg):
+ """ Convert a C printf()-style template to a pattern suitable for fwts """
+
+ # Somewhat-simplified match for a %-template
+ template_re = re.compile(
+ '%(?P<flag>[-#0 +]*)'
+ '(?P<width>(?:[1-9][0-9]*|\*))?'
+ '(?P<precision>\.*(?:[1-9][0-9]*|\*))?'
+ '(?:hh|h|l|ll|L|j|z|t)?'
+ '(?P<conversion>[a-zA-Z%])')
+ global is_regex
+ is_regex = False
+
+ def expand_template(match):
+ global is_regex
+ c = match.group('conversion').lower()
+ if c == '%':
+ return '%'
+ is_regex = True
+ if c in ['d', 'i', 'u']:
+ return '[0-9]+'
+ elif c == 'o':
+ return '[0-7]+'
+ elif c == 'x':
+ return '[0-9a-f]+'
+ elif c == 'p':
+ return '(?:0x[0-9a-f]+|nil)'
+ elif c == 's':
+ return '.*'
+ else:
+ warn(loc, "Unknown template conversion '%s'" % match.group(0))
+ return '.*'
+
+ escape_re = re.compile(r'\\(?P<char>.)', re.DOTALL)
+ def expand_escape(match):
+ global is_regex
+ c = match.group('char')
+ if c == 'n':
+ return '\n'
+ elif c in ['\\', '"']:
+ return c
+ else:
+ warn(loc, "Unhandled escape sequence '%s'" % match.group(0))
+ is_regex = True
+ return '.'
+
+ pattern = template_re.sub(expand_template, msg)
+ pattern = escape_re.sub(expand_escape, pattern)
+ pattern = pattern.strip()
+
+ compare_mode = "string"
+ if is_regex:
+ compare_mode = "regex"
+
+ return (compare_mode, pattern)
+
+def parse_patterns(parser, fname):
+ patterns = []
+ data = open(fname).read()
+ i = 1
+ for result in parser.scanString(data):
+ (token, loc, _) = result
+ (annotations, logfn, level, msg) = token
+
+ loc = (fname, lineno(loc, data))
+
+ if logfn != 'prlog':
+ warn(loc, "unknown log output function '%s'" % logfn)
+
+ compare_mode, pattern_str = message_to_pattern(loc, msg)
+
+ pattern = {
+ 'log_level': log_level_to_fwts(level),
+ 'compare_mode': compare_mode,
+ 'pattern': pattern_str,
+ }
+
+ pattern.update(annotations)
+
+ if not 'label' in pattern:
+ warn(loc, "missing label")
+ pattern['label'] = '%s:%d' % (fname, i)
+ i += 1
+
+ if not 'advice' in pattern:
+ warn(loc, "missing advice")
+
+ allowed_data = ['compare_mode', 'log_level',
+ 'pattern', 'advice', 'label']
+ extras = set(pattern.keys()) - set(allowed_data)
+ if extras:
+ warn(loc, "unknown pattern annotation: %s" %
+ ','.join([ "'%s'" % e for e in extras]))
+ for e in extras:
+ del pattern[e]
+
+ patterns.append(pattern)
+
+ return patterns
+
+if __name__ == '__main__':
+ argparser = argparse.ArgumentParser(
+ description='Generate FWTS olog definitions from the skiboot '
+ 'source tree')
+ argparser.add_argument('directories', metavar='DIR', nargs='*',
+ help='path to source files (default .)', default=['.'])
+ argparser.add_argument('--output', '-o', metavar='FILE',
+ type=argparse.FileType('w'), default=sys.stdout,
+ help='output to FILE (default to stdout)', nargs='?')
+ args = argparser.parse_args()
+
+ sources = []
+ for directory in args.directories:
+ sources.extend(find_sources(directory))
+
+ parser = create_parser()
+ patterns = []
+ for source in sources:
+ patterns.extend(parse_patterns(parser, source))
+
+ data = {'olog_error_warning_patterns': patterns}
+
+ args.output.write(json.dumps(data, **json_params) + '\n')
+
--
2.5.0
More information about the Skiboot
mailing list