diff options
author | Corey Farrell <git@cfware.com> | 2014-06-27 19:18:32 +0000 |
---|---|---|
committer | Corey Farrell <git@cfware.com> | 2014-06-27 19:18:32 +0000 |
commit | 9a495107b86b80a4bd804b2ddd7c4d7997ee6a3c (patch) | |
tree | 74c552f0d69b82de7b7329270b6a91177e004e45 /contrib/scripts/refcounter.py | |
parent | 15dcaeef82ce4fc013668392838f8ff9e0ec8075 (diff) |
refcounter.py: prevent use of excessive RAM with large refs logs
When processing a 212MB refs file, refcounter.py used over 3GB of RAM.
This change greatly reduces memory usage in two ways:
* Saving object history in whole lines instead of separated values.
* Not saving normal/skewed/leaked object lists unless they are requested.
ASTERISK-23921 #close
Reported by: Corey Farrell
Review: https://reviewboard.asterisk.org/r/3668/
........
Merged revisions 417480 from http://svn.asterisk.org/svn/asterisk/branches/1.8
........
Merged revisions 417481 from http://svn.asterisk.org/svn/asterisk/branches/11
........
Merged revisions 417483 from http://svn.asterisk.org/svn/asterisk/branches/12
git-svn-id: https://origsvn.digium.com/svn/asterisk/trunk@417485 65c4cc65-6c06-0410-ace0-fbb531ad65f3
Diffstat (limited to 'contrib/scripts/refcounter.py')
-rwxr-xr-x | contrib/scripts/refcounter.py | 47 |
1 files changed, 29 insertions, 18 deletions
diff --git a/contrib/scripts/refcounter.py b/contrib/scripts/refcounter.py index cd4d69f46..3aad3530a 100755 --- a/contrib/scripts/refcounter.py +++ b/contrib/scripts/refcounter.py @@ -23,6 +23,7 @@ import os from optparse import OptionParser + def parse_line(line): """Parse out a line into its constituent parts. @@ -46,11 +47,11 @@ def parse_line(line): 'function': tokens[5], 'state': tokens[6], 'tag': tokens[7], - } + } return processed_line -def process_file(filename): +def process_file(options): """The routine that kicks off processing a ref file Keyword Arguments: @@ -67,6 +68,7 @@ def process_file(filename): leaked_objects = [] skewed_objects = [] current_objects = {} + filename = options.filepath with open(filename, 'r') as ref_file: for line in ref_file: @@ -78,16 +80,25 @@ def process_file(filename): if obj not in current_objects: current_objects[obj] = [] - if 'constructor' not in parsed_line['state']: - skewed_objects.append(current_objects[obj]) - current_objects[obj].append(parsed_line) + if options.skewed and 'constructor' not in parsed_line['state']: + skewed_objects.append((obj, current_objects[obj])) + current_objects[obj].append("[%s] %s:%s %s: %s %s - [%s]" % ( + parsed_line['thread_id'], + parsed_line['file'], + parsed_line['line'], + parsed_line['function'], + parsed_line['delta'], + parsed_line['tag'], + parsed_line['state'])) if 'destructor' in parsed_line['state']: - lifetime = current_objects.get(obj) - finished_objects.append(lifetime) + if options.normal: + finished_objects.append((obj, current_objects[obj])) del current_objects[obj] - leaked_objects = current_objects.values() + if options.leaks: + for key, lines in current_objects.iteritems(): + leaked_objects.append((key, lines)) return (finished_objects, leaked_objects, skewed_objects) @@ -103,13 +114,9 @@ def print_objects(objects, prefix=""): print "======== %s Objects ========" % prefix print "\n" for obj in objects: - print "==== %s Object %s history ====" % (prefix, obj[0]['addr']) - for entry in obj: - print "[%s] %s:%s %s: %s %s - [%s]" % (entry['thread_id'], - entry['file'], entry['line'], - entry['function'], - entry['delta'], entry['tag'], - entry['state']) + print "==== %s Object %s history ====" % (prefix, obj[0]) + for line in obj[1]: + print line print "\n" @@ -131,15 +138,19 @@ def main(argv=None): help="If specified, don't output leaked objects") parser.add_option("-n", "--suppress-normal", action="store_false", dest="normal", default=True, - help="If specified, don't output objects with a " \ + help="If specified, don't output objects with a " "complete lifetime") parser.add_option("-s", "--suppress-skewed", action="store_false", dest="skewed", default=True, - help="If specified, don't output objects with a " \ + help="If specified, don't output objects with a " "skewed lifetime") (options, args) = parser.parse_args(argv) + if not options.leaks and not options.skewed and not options.normal: + print >>sys.stderr, "All options disabled" + return -1 + if not os.path.isfile(options.filepath): print >>sys.stderr, "File not found: %s" % options.filepath return -1 @@ -147,7 +158,7 @@ def main(argv=None): try: (finished_objects, leaked_objects, - skewed_objects) = process_file(options.filepath) + skewed_objects) = process_file(options) if options.leaks and len(leaked_objects): print_objects(leaked_objects, "Leaked") |