Hello All, I keep coming across a memory error when processing many netcdf files. I assume it has something to do with how I loop things and maybe need to close things off properly. In the code below I am looping through a bunch of netcdf files (each file is hourly data for one month) and within each netcdf file I am outputting a *png file every three hours. This works for one netcdf file but when it begins to process the next netcdf file I receive this memory error:
*Traceback (most recent call last): File "d:/plot_netcdf_merc_multiplot_across_multifolders_mkdirs_memoryerror.py", line 44, in <module> TSFC=ncfile.variables['T_SFC'][:] File "netCDF4.pyx", line 2473, in netCDF4.Variable.__getitem__ (netCDF4.c:23094) MemoryError* To reduce processing requirements I have tried making the LAT and LON to only use [0] but I also receive an error: *Traceback (most recent call last): File "d:/plot_netcdf_merc_multiplot_across_multifolders_mkdirs_memoryerror.py", line 75, in <module> x,y=map(*N.meshgrid(LON,LAT)) File "C:\Python27\lib\site-packages\numpy\lib\function_base.py", line 3256, in meshgrid numRows, numCols = len(y), len(x) # yes, reversed TypeError: len() of unsized object* finally I have added gc.collect() in a couple of places but that doesn't seem to do anything to help. I am using :*Python 2.7.2 |EPD 7.1-2 (32-bit)| (default, Jul 3 2011, 15:13:59) [MSC v.1500 32 bit (Intel)] on win32* Any feedback will be greatly appreciated! from netCDF4 import Dataset import numpy import numpy as N import matplotlib.pyplot as plt from numpy import ma as MA from mpl_toolkits.basemap import Basemap from netcdftime import utime from datetime import datetime import os import gc print "start processing...." inputpath=r'E:/GriddedData/Input/' outputpath=r'E:/GriddedData/Validation/' shapefile1="E:/test_GIS/DSE_REGIONS" for (path, dirs, files) in os.walk(inputpath): for dir in dirs: print dir sourcepath=os.path.join(path,dir) relativepath=os.path.relpath(sourcepath,inputpath) newdir=os.path.join(outputpath,relativepath) if not os.path.exists(newdir): os.makedirs(newdir) for ncfile in files: if ncfile[-3:]=='.nc': print "dealing with ncfiles:", ncfile ncfile=os.path.join(sourcepath,ncfile) #print ncfile ncfile=Dataset(ncfile, 'r+', 'NETCDF4') TSFC=ncfile.variables['T_SFC'][:,:,:] TIME=ncfile.variables['time'][:] LAT=ncfile.variables['latitude'][:] LON=ncfile.variables['longitude'][:] fillvalue=ncfile.variables['T_SFC']._FillValue TSFC=MA.masked_values(TSFC, fillvalue) ncfile.close() gc.collect() print "garbage collected" for TSFC, TIME in zip((TSFC[1::3]),(TIME[1::3])): print TSFC, TIME #convert time from numbers to date and prepare it to have no symbols for saving to filename cdftime=utime('seconds since 1970-01-01 00:00:00') ncfiletime=cdftime.num2date(TIME) print ncfiletime timestr=str(ncfiletime) d = datetime.strptime(timestr, '%Y-%m-%d %H:%M:%S') date_string = d.strftime('%Y%m%d_%H%M') #Set up basemap using mercator projection http://matplotlib.sourceforge.net/basemap/doc/html/users/merc.html map = Basemap(projection='merc',llcrnrlat=-40,urcrnrlat=-33, llcrnrlon=139.0,urcrnrlon=151.0,lat_ts=0,resolution='i') # compute map projection coordinates for lat/lon grid. x,y=map(*N.meshgrid(LON,LAT)) map.drawcoastlines(linewidth=0.5) map.readshapefile(shapefile1, 'DSE_REGIONS') map.drawstates() plt.title('Surface temperature at %s UTC'%ncfiletime) ticks=[-5,0,5,10,15,20,25,30,35,40,45,50] CS = map.contourf(x,y,TSFC, ticks, cmap=plt.cm.jet) l,b,w,h =0.1,0.1,0.8,0.8 cax = plt.axes([l+w+0.025, b, 0.025, h], ) cbar=plt.colorbar(CS, cax=cax, drawedges=True) #save map as *.png and plot netcdf file plt.savefig((os.path.join(newdir,'TSFC'+date_string+'UTC.png'))) plt.close() gc.collect() print "garbage collected again" print "end of processing"
-- http://mail.python.org/mailman/listinfo/python-list