Loop code # TODO: figure out memory issuefunction train!(m::Model, s::Adagrad; xmax=100, alpha=0.75) J = 0.0 shuffle!(m.covec) vecsize = size(m.W_main, 1) eltype = typeof(m.b_main[1]) vm = zeros(eltype, vecsize) vc = zeros(eltype, vecsize) grad_main = zeros(eltype, vecsize) grad_ctx = zeros(eltype, vecsize) for n=1:s.niter # shuffle indices for i = 1:length(m.covec) @inbounds l1 = m.covec[i].i # main index @inbounds l2 = m.covec[i].j # context index @inbounds v = m.covec[i].v #= vm[:] = m.W_main[:, l1] =# #= vc[:] = m.W_ctx[:, l2] =# @inbounds for j = 1:vecsize vm[j] = m.W_main[j, l1] vc[j] = m.W_ctx[j, l2] end diff = dot(vec(vm), vec(vc)) + m.b_main[l1] + m.b_ctx[l2] - log(v) fdiff = ifelse(v < xmax, (v / xmax) ^ alpha, 1.0) * diff J += 0.5 * fdiff * diff fdiff *= s.lrate # inc memory by ~200 MB && running time by 2x #= grad_main[:] = fdiff * m.W_ctx[:, l2] =# #= grad_ctx[:] = fdiff * m.W_main[:, l1] =# @inbounds for j = 1:vecsize grad_main[j] = fdiff * m.W_ctx[j, l2] grad_ctx[j] = fdiff * m.W_main[j, l1] end # Adaptive learning # inc ~ 600MB + 0.75s #= m.W_main[:, l1] -= grad_main ./ sqrt(m.W_main_grad[:, l1]) =# #= m.W_ctx[:, l2] -= grad_ctx ./ sqrt(m.W_ctx_grad[:, l2]) =# #= m.b_main[l1] -= fdiff ./ sqrt(m.b_main_grad[l1]) =# #= m.b_ctx[l2] -= fdiff ./ sqrt(m.b_ctx_grad[l2]) =# @inbounds for j = 1:vecsize m.W_main[j, l1] -= grad_main[j] / sqrt(m.W_main_grad[j, l1]) m.W_ctx[j, l2] -= grad_ctx[j] / sqrt(m.W_ctx_grad[j, l2]) end m.b_main[l1] -= fdiff ./ sqrt(m.b_main_grad[l1]) m.b_ctx[l2] -= fdiff ./ sqrt(m.b_ctx_grad[l2]) # Gradients fdiff *= fdiff #= m.W_main_grad[:, l1] += grad_main .^ 2 =# #= m.W_ctx_grad[:, l2] += grad_ctx .^ 2 =# #= m.b_main_grad[l1] += fdiff =# #= m.b_ctx_grad[l2] += fdiff =# @inbounds for j = 1:vecsize m.W_main_grad[j, l1] += grad_main[j] ^ 2 m.W_ctx_grad[j, l2] += grad_ctx[j] ^ 2 end m.b_main_grad[l1] += fdiff m.b_ctx_grad[l2] += fdiff end #= if n % 10 == 0 =# #= println("iteration $n, cost $J") =# #= end =# endend
Mixmax [https://emailapps.mixmax.com/img/badge_mixmax.png] [https://mixmax.com/r/YQkS9vBkBR3wSpESz] Not using Mixmax yet? [https://mixmax.com/r/YQkS9vBkBR3wSpESz] And the respective timings @time GloVe.train!(model, GloVe.Adagrad(500)) 7.097 seconds (96237 k allocations: 1468 MB, 7.01% gc time) Slower and more memory. On Sun, May 24, 2015 at 4:21 AM, Mauro < mauro...@runbox.com [mauro...@runbox.com] > wrote: Loops should run without allocations. Can you post your loop-code? > A[i, :] = 0.5 * B[i, :] To state the obvious, as loop: for j=1:size(A,2) A[i,j] = 0.5 * B[i,j] end this shouldn't allocate, if i is an integer. Unless A and B have different type, then allocation might happen. On Sun, 2015-05-24 at 05:00, Dom Luna < dluna...@gmail.com [dluna...@gmail.com] > wrote: > Reposting this from Gitter chat since it seems this is more active. > > I'm writing a GloVe module to learn Julia. > > How can I avoid memory allocations? My main function deals with a lot of > random indexing in Matrices. > > A[i, :] = 0.5 * B[i, :] > > In this case* i* isn't from a linear sequence. I'm not sure that matters. > Anyway, I’ve done analysis and I know B[i, :] is the issue here since it’s > creating a copy. > > https://github.com/JuliaLang/ julia/blob/master/base/array. jl#L309 [https://github.com/JuliaLang/julia/blob/master/base/array.jl#L309] makes the > copy > > > I tried to do it via loop but it looks like that doesn’t help either. In > fact, it seems to allocate slight more memory which seems really odd. > > Here’s some of the code, it’s a little messy since I’m commenting different > approaches I’m trying out. > > type Model{T} > W_main::Matrix{T} > W_ctx::Matrix{T} > b_main::Vector{T} > b_ctx::Vector{T} > W_main_grad::Matrix{T} > W_ctx_grad::Matrix{T} > b_main_grad::Vector{T} > b_ctx_grad::Vector{T} > covec::Vector{Cooccurence} > end > > # Each vocab word in associated with a main vector and a context vector. > # The paper initializes the to values [-0.5, 0.5] / vecsize+1 and > # the gradients to 1.0. > # > # The +1 term is for the bias. > function Model(comatrix; vecsize=100) > vs = size(comatrix, 1) > Model( > (rand(vecsize, vs) - 0.5) / (vecsize + 1), > (rand(vecsize, vs) - 0.5) / (vecsize + 1), > (rand(vs) - 0.5) / (vecsize + 1), > (rand(vs) - 0.5) / (vecsize + 1), > ones(vecsize, vs), > ones(vecsize, vs), > ones(vs), > ones(vs), > CoVector(comatrix), # not required in 0.4 > ) > end > > # TODO: figure out memory issue > # the memory comments are from 500 loop test with vecsize=100 > function train!(m::Model, s::Adagrad; xmax=100, alpha=0.75) > J = 0.0 > shuffle!(m.covec) > > vecsize = size(m.W_main, 1) > eltype = typeof(m.b_main[1]) > vm = zeros(eltype, vecsize) > vc = zeros(eltype, vecsize) > grad_main = zeros(eltype, vecsize) > grad_ctx = zeros(eltype, vecsize) > > for n=1:s.niter > # shuffle indices > for i = 1:length(m.covec) > @inbounds l1 = m.covec[i].i # main index > @inbounds l2 = m.covec[i].j # context index > @inbounds v = m.covec[i].v > > vm[:] = m.W_main[:, l1] > vc[:] = m.W_ctx[:, l2] > > diff = dot(vec(vm), vec(vc)) + m.b_main[l1] + m.b_ctx[l2] - > log(v) > fdiff = ifelse(v < xmax, (v / xmax) ^ alpha, 1.0) * diff > J += 0.5 * fdiff * diff > > fdiff *= s.lrate > # inc memory by ~200 MB && running time by 2x > grad_main[:] = fdiff * m.W_ctx[:, l2] > grad_ctx[:] = fdiff * m.W_main[:, l1] > > # Adaptive learning > # inc ~ 600MB + 0.75s > #= @inbounds for ii = 1:vecsize =# > #= m.W_main[ii, l1] -= grad_main[ii] / > sqrt(m.W_main_grad[ii, l1]) =# > #= m.W_ctx[ii, l2] -= grad_ctx[ii] / sqrt(m.W_ctx_grad[ii, > l2]) =# > #= m.b_main[l1] -= fdiff ./ sqrt(m.b_main_grad[l1]) =# > #= m.b_ctx[l2] -= fdiff ./ sqrt(m.b_ctx_grad[l2]) =# > #= end =# > > m.W_main[:, l1] -= grad_main ./ sqrt(m.W_main_grad[:, l1]) > m.W_ctx[:, l2] -= grad_ctx ./ sqrt(m.W_ctx_grad[:, l2]) > m.b_main[l1] -= fdiff ./ sqrt(m.b_main_grad[l1]) > m.b_ctx[l2] -= fdiff ./ sqrt(m.b_ctx_grad[l2]) > > # Gradients > fdiff *= fdiff > m.W_main_grad[:, l1] += grad_main .^ 2 > m.W_ctx_grad[:, l2] += grad_ctx .^ 2 > m.b_main_grad[l1] += fdiff > m.b_ctx_grad[l2] += fdiff > end > > #= if n % 10 == 0 =# > #= println(“iteration $n, cost $J”) =# > #= end =# > end > end > > > Here’s the entire repo https://github.com/domluna/ GloVe.jl > [https://github.com/domluna/GloVe.jl] . Might be > helpful. > > I tried doing some loops but it allocates more memory (oddly enough) and > gets slower. > > You’ll notice the word vectors are indexed by column, I changed the > representation to that > seeing if it would make a difference during the loop. It didn’t seem to. > > The memory analysis showed > > Julia Version 0.4.0-dev+4893 > Commit eb5da26* (2015-05-19 11:51 UTC) > Platform Info: > System: Darwin (x86_64-apple-darwin14.4.0) > CPU: Intel(R) Core(TM) i5-2557M CPU @ 1.70GHz > WORD_SIZE: 64 > BLAS: libopenblas (USE64BITINT DYNAMIC_ARCH NO_AFFINITY Sandybridge) > LAPACK: libopenblas > LIBM: libopenlibm > LLVM: libLLVM-3.3 > > Here model consists of 100x19 Matrices and 100 element vectors, 19 words in > the vocab, 100 element word vector. > > @time GloVe.train!(model, GloVe.Adagrad(500)) > 1.990 seconds (6383 k allocations: 1162 MB, 10.82% gc time) > > 0.3 has is a bit slower due to worse gc but same memory. > > Any help would be greatly appreciated! cheers, dom Sent with Mixmax [https://mixmax.com] [https://app.mixmax.com/api/track/v2/kcSBzxMuxJTn4TZ8M/dluna132%40gmail.com/i02bj5ycwV3bydWZsd2bvdGQzJXZzVXLhlGb1pmI/i02bj5ycwV3bydWZsd2bvdGQzJXZzVXLhlGb1pmI?sc=false]