It's odd that two people would notice the same problem at the same time on old branches.
Anyway, I'm attaching the scripts I have. They are rough. The second one invokes the first one as a subprocess; it is probably the one you should use. I might have to walk you through how to use it, or write better documentation myself. Anyway, it should be a start. On Wed, Feb 20, 2019 at 07:15:26PM +0400, Oleg Bondarev wrote: > Ah, sorry, I missed "ovs-vswitchd memory consumption behavior" thread. > So I guess I'm also interested in the scripts for analyzing the heap in a > core dump :) > > Thanks, > Oleg > > On Wed, Feb 20, 2019 at 7:00 PM Oleg Bondarev <obonda...@mirantis.com> > wrote: > > > Hi, > > > > OVS 2.8.0, uptime 197 days, 44G RAM. > > ovs-appctl memory/show reports: > > "handlers:35 ofconns:4 ports:73 revalidators:13 rules:1099 udpif > > keys:686" > > > > Similar data on other nodes of the OpenStack cluster. > > Seems usage grows gradually over time. > > Are there any known issues, like > > http://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2017-14970? > > Please advise on the best way to debug. > > > > Thanks, > > Oleg > > > > > _______________________________________________ > discuss mailing list > disc...@openvswitch.org > https://mail.openvswitch.org/mailman/listinfo/ovs-discuss
#! /usr/bin/perl use strict; #use warnings; use Fcntl; my ($core, $start, $len, $vma) = @ARGV; $start = oct($start) if $start =~ /^0/; $len = oct($len) if $len =~ /^0/; $vma = oct($vma) if defined($vma) && ($vma =~ /^0/); open (CORE, '<', $core) or die "$core: open: $!\n"; my $bits = 64; my $bytes = $bits / 8; my $template; if ($bits == 64) { $template = 'QQ'; } elsif ($bits == 32) { $template = 'LL'; } else { die; } seek (CORE, $start, Fcntl::SEEK_SET) or die "$core: seek: $!\n"; my $ofs = $start; my $n = 0; my $n_inuse = 0; my $last_size = 0; my $last_ofs = 0; my $bytes_inuse = 0; my %hist; my %loc; while ($len > $bytes * 2) { my $header; read (CORE, $header, $bytes * 2) or die "$core: read: $!\n"; $ofs += $bytes * 2; my ($prev_size, $size) = unpack($template, $header); #print "$prev_size, $size\n"; die if $size < $bytes * 2; die "$size byte block at offset $ofs" if $size > 10485760; $n++; if ($size & 1) { #printf "0x%x, %d used\n", ($last_ofs - $start) + $vma, $last_size - $bytes * 2; $n_inuse++; my $sample = !$hist{$last_size}++ || int(rand($hist{$last_size})) < 1; $loc{$last_size} = $last_ofs if $sample; $bytes_inuse += $last_size; } else { #printf "0x%x, %u free\n", ($last_ofs - $start) + $vma, $last_size - $bytes * 2; } $last_size = $size & ~7; $last_ofs = $ofs; $len -= $size & ~7; my $content_len = ($size & ~7) - $bytes * 2; my $content; read (CORE, $content, $content_len); $ofs += $content_len; } print "$n blocks, $n_inuse blocks in use, $bytes_inuse bytes in use\n"; print "In use:\n"; foreach my $size (sort { $a <=> $b }(keys(%hist))) { my $realsize = $size - 2 * $bytes; printf "%6d bytes x %6d = %10d (0x%x)\n", $realsize, $hist{$size}, $realsize * $hist{$size}, $loc{$size}; if ($realsize == 96) { printf "starting at vaddr 0x%x:\n", $loc{$size} - $start + $vma; system("hd -n $realsize -s $loc{$size} $core"); } }
#! /usr/bin/perl use strict; #use warnings; use Fcntl; my ($core) = @ARGV; # Notes: # # - Each heap begins with a 'heap_info' structure, which is 32 bytes long # on 64-bit systems: # # typedef struct _heap_info # { # mstate ar_ptr; /* Arena for this heap. */ # struct _heap_info *prev; /* Previous heap. */ # size_t size; /* Current size in bytes. */ # size_t mprotect_size; /* Size in bytes that has been mprotected # PROT_READ|PROT_WRITE. */ # /* Make sure the following data is properly aligned, particularly # that sizeof (heap_info) + 2 * SIZE_SZ is a multiple of # MALLOC_ALIGNMENT. */ # char pad[-6 * SIZE_SZ & MALLOC_ALIGN_MASK]; # } heap_info; # # - heap_info.ar_ptr points to a 'struct malloc_state' (typedefed to mstate), # which is a 2192-byte structure (on 64-bit systems): # # struct malloc_state # { # /* Serialize access. */ # __libc_lock_define (, mutex); # # /* Flags (formerly in max_fast). */ # int flags; # # /* Set if the fastbin chunks contain recently inserted free blocks. */ # /* Note this is a bool but not all targets support atomics on booleans. */ # int have_fastchunks; # # /* Fastbins */ # mfastbinptr fastbinsY[NFASTBINS]; # # /* Base of the topmost chunk -- not otherwise kept in a bin */ # mchunkptr top; # # /* The remainder from the most recent split of a small request */ # mchunkptr last_remainder; # # /* Normal bins packed as described above */ # mchunkptr bins[NBINS * 2 - 2]; # # /* Bitmap of bins */ # unsigned int binmap[BINMAPSIZE]; # # /* Linked list */ # struct malloc_state *next; # # /* Linked list for free arenas. Access to this field is serialized # by free_list_lock in arena.c. */ # struct malloc_state *next_free; # # /* Number of threads attached to this arena. 0 if the arena is on # the free list. Access to this field is serialized by # free_list_lock in arena.c. */ # INTERNAL_SIZE_T attached_threads; # # /* Memory allocated from the system in this arena. */ # INTERNAL_SIZE_T system_mem; # INTERNAL_SIZE_T max_system_mem; # }; # # - A chunk is in the main arena if its 'A' bit is 0, in which case its arena # is the static variable 'main_arena'. # # - A chunk is in a non-main arena if its 'A' bit is 1, in which case its # heap_info can be found by clearing several low-order bits in the address, # then the arena can be found by following ar_ptr from heap_info: # # #define heap_for_ptr(ptr) \ # ((heap_info *) ((unsigned long) (ptr) & ~(HEAP_MAX_SIZE - 1))) # #define arena_for_chunk(ptr) \ # (chunk_main_arena (ptr) ? &main_arena : heap_for_ptr (ptr)->ar_ptr) # # - The size of the block allocated for a given requested size # (including the internal overhead) is calculated with request2size. # # #define request2size(req) \ # (((req) + SIZE_SZ + MALLOC_ALIGN_MASK < MINSIZE) ? \ # MINSIZE : \ # ((req) + SIZE_SZ + MALLOC_ALIGN_MASK) & ~MALLOC_ALIGN_MASK) # # where SIZE_SZ is 8, MALLOC_ALIGN_MASK is 15, and MINSIZE is 32. In effect, # this is MAX(32, ROUND_UP(req + 16, 16)). # # /* The smallest possible chunk */ # #define MIN_CHUNK_SIZE (offsetof(struct malloc_chunk, fd_nextsize)) # # /* The smallest size we can malloc is an aligned minimal chunk */ # # #define MINSIZE \ # (unsigned long)(((MIN_CHUNK_SIZE+MALLOC_ALIGN_MASK) & ~MALLOC_ALIGN_MASK)) # /* The corresponding word size. */ # #define SIZE_SZ (sizeof (INTERNAL_SIZE_T)) # # /* The corresponding bit mask value. */ # #define MALLOC_ALIGN_MASK (MALLOC_ALIGNMENT - 1) # # struct malloc_chunk { # # INTERNAL_SIZE_T mchunk_prev_size; /* Size of previous chunk (if free). */ # INTERNAL_SIZE_T mchunk_size; /* Size in bytes, including overhead. */ # # struct malloc_chunk* fd; /* double links -- used only if free. */ # struct malloc_chunk* bk; # # /* Only used for large blocks: pointer to next larger size. */ # struct malloc_chunk* fd_nextsize; /* double links -- used only if free. */ # struct malloc_chunk* bk_nextsize; # }; # - MALLOC_ALIGNMENT is 16. open (OBJDUMP, "objdump -p '$core'|"); while (<OBJDUMP>) { next unless /LOAD\s+off/; my $hex = '(0x[0-9a-fA-F]+)'; my ($off, $vaddr) = /off\s+$hex\s+vaddr\s+$hex\s+/ or die; $_ = <OBJDUMP>; my ($filesz, $memsz, $flags) = /filesz\s+$hex\s+memsz\s+$hex\s+flags\s+(...)/ or die; next if $flags ne 'rw-'; next if $filesz ne $memsz; print "analyzing $filesz bytes starting at $off:\n"; system("dump-heap $core $off $filesz $vaddr"); system("dump-heap $core ". (hex($off) + 32) . " " . (hex($filesz) - 32) . " " . (hex($vaddr) + 32)); } exit 0;
_______________________________________________ discuss mailing list disc...@openvswitch.org https://mail.openvswitch.org/mailman/listinfo/ovs-discuss