Re: lockup in bgp_delete()
On 3/20/17 6:29 PM, Eduardo Bustamante wrote:
> This was reported a month ago:
> http://lists.gnu.org/archive/html/bug-bash/2017-02/msg00025.html
The devel git branch on savannah has several fixes for this. If you
don't want to run that on a server, you can just snag the jobs.c file
from the latest snapshot and either try to just drop it in, or see
if the attached patch works.
Chet
--
``The lyf so short, the craft so long to lerne.'' - Chaucer
``Ars longa, vita brevis'' - Hippocrates
Chet Ramey, UTech, CWRUc...@case.eduhttp://cnswww.cns.cwru.edu/~chet/
*** ../bash-4.4-patched/jobs.c 2016-11-11 13:42:55.0 -0500
--- jobs.c 2017-02-22 15:16:28.0 -0500
***
*** 813,818
struct pidstat *ps;
! bucket = pshash_getbucket (pid);
! psi = bgp_getindex ();
ps = &bgpids.storage[psi];
--- 796,815
struct pidstat *ps;
! /* bucket == existing chain of pids hashing to same value
! psi = where were going to put this pid/status */
!
! bucket = pshash_getbucket (pid); /* index into pidstat_table */
! psi = bgp_getindex (); /* bgpids.head, index into storage */
!
! /* XXX - what if psi == *bucket? */
! if (psi == *bucket)
! {
! #ifdef DEBUG
! internal_warning ("hashed pid %d (pid %d) collides with bgpids.head, skipping", psi, pid);
! #endif
! bgpids.storage[psi].pid = NO_PID; /* make sure */
! psi = bgp_getindex (); /* skip to next one */
! }
!
ps = &bgpids.storage[psi];
***
*** 842,845
--- 839,843
{
struct pidstat *ps;
+ ps_index_t *bucket;
ps = &bgpids.storage[psi];
***
*** 847,856
return;
! if (ps->bucket_next != NO_PID)
bgpids.storage[ps->bucket_next].bucket_prev = ps->bucket_prev;
! if (ps->bucket_prev != NO_PID)
bgpids.storage[ps->bucket_prev].bucket_next = ps->bucket_next;
else
! *(pshash_getbucket (ps->pid)) = ps->bucket_next;
}
--- 845,861
return;
! if (ps->bucket_next != NO_PIDSTAT)
bgpids.storage[ps->bucket_next].bucket_prev = ps->bucket_prev;
! if (ps->bucket_prev != NO_PIDSTAT)
bgpids.storage[ps->bucket_prev].bucket_next = ps->bucket_next;
else
! {
! bucket = pshash_getbucket (ps->pid);
! *bucket = ps->bucket_next; /* deleting chain head in hash table */
! }
!
! /* clear out this cell, just in case */
! ps->pid = NO_PID;
! ps->bucket_next = ps->bucket_prev = NO_PIDSTAT;
}
***
*** 859,863
pid_t pid;
{
! ps_index_t psi;
if (bgpids.storage == 0 || bgpids.nalloc == 0 || bgpids.npid == 0)
--- 864,868
pid_t pid;
{
! ps_index_t psi, orig_psi;
if (bgpids.storage == 0 || bgpids.nalloc == 0 || bgpids.npid == 0)
***
*** 865,871
/* Search chain using hash to find bucket in pidstat_table */
! for (psi = *(pshash_getbucket (pid)); psi != NO_PIDSTAT; psi = bgpids.storage[psi].bucket_next)
! if (bgpids.storage[psi].pid == pid)
! break;
if (psi == NO_PIDSTAT)
--- 870,883
/* Search chain using hash to find bucket in pidstat_table */
! for (orig_psi = psi = *(pshash_getbucket (pid)); psi != NO_PIDSTAT; psi = bgpids.storage[psi].bucket_next)
! {
! if (bgpids.storage[psi].pid == pid)
! break;
! if (orig_psi == bgpids.storage[psi].bucket_next) /* catch reported bug */
! {
! internal_warning ("bgp_delete: LOOP: psi (%d) == storage[psi].bucket_next", psi);
! return 0;
! }
! }
if (psi == NO_PIDSTAT)
***
*** 905,909
pid_t pid;
{
! ps_index_t psi;
if (bgpids.storage == 0 || bgpids.nalloc == 0 || bgpids.npid == 0)
--- 917,921
pid_t pid;
{
! ps_index_t psi, orig_psi;
if (bgpids.storage == 0 || bgpids.nalloc == 0 || bgpids.npid == 0)
***
*** 911,917
/* Search chain using hash to find bucket in pidstat_table */
! for (psi = *(pshash_getbucket (pid)); psi != NO_PIDSTAT; psi = bgpids.storage[psi].bucket_next)
! if (bgpids.storage[psi].pid == pid)
! return (bgpids.storage[psi].status);
return -1;
--- 923,936
/* Search chain using hash to find bucket in pidstat_table */
! for (orig_psi = psi = *(pshash_getbucket (pid)); psi != NO_PIDSTAT; psi = bgpids.storage[psi].bucket_next)
! {
! if (bgpids.storage[psi].pid == pid)
! return (bgpids.storage[psi].status);
! if (orig_psi == bgpids.storage[psi].bucket_next) /* catch reported bug */
! {
! internal_warning ("bgp_search: LOOP: psi (%d) == storage[psi].bucket_next", psi);
! return -1;
! }
! }
return -1;
arithmetic syntax error loop after declaration of exported integer variable
I get an infinite error loop in B, and I suppose that's not because of my settings. # A bash$ unset -v foo; declare -i foo; foo='bar bar' bash: bar bar: syntax error in expression (error token is "bar") # B bash$ unset -v foo; foo='bar bar' declare -x -i foo [...]
Re: lockup in bgp_delete()
This was reported a month ago: http://lists.gnu.org/archive/html/bug-bash/2017-02/msg00025.html
lockup in bgp_delete()
Configuration Information [Automatically generated, do not change]: Machine: x86_64 OS: linux-gnu Compiler: gcc Compilation CFLAGS: -DPROGRAM='bash' -DCONF_HOSTTYPE='x86_64' -DCONF_OSTYPE='linux-gnu' -DCONF_MACHTYPE='x86_64-unknown-linux-gnu' -DCONF_VENDOR='unknown' -DLOCALEDIR='/usr/share/locale' -DPACKAGE='bash' -DSHELL -DHAVE_CONFIG_H -I. -I. -I./include -I./lib -D_FORTIFY_SOURCE=2 -march=x86-64 -mtune=generic -O2 -pipe -fstack-protector-strong -DDEFAULT_PATH_VALUE='/usr/local/sbin:/usr/local/bin:/usr/bin' -DSTANDARD_UTILS_PATH='/usr/bin' -DSYS_BASHRC='/etc/bash.bashrc' -DSYS_BASH_LOGOUT='/etc/bash.bash_logout' -DNON_INTERACTIVE_LOGIN_SHELLS -Wno-parentheses -Wno-format-security uname output: Linux build64-par 4.9.11-gnu-1 #1 SMP PREEMPT Sun Feb 19 18:36:28 UYT 2017 x86_64 GNU/Linux Machine Type: x86_64-unknown-linux-gnu Bash Version: 4.4 Patch Level: 12 Release Status: release Description: Occasionally, on one of my servers, a bash script in a cron job locks up, and pegs one of the CPU cores at 100%. Attaching to it with GDB, I see that it is stuck in the loop in bgp_delete; it is looking for the index of the pid to delete in bgpids.storage, but it's not there. And bgpids.storage being a circular linked list, it just loops around and around, never exiting the loop. Repeat-By: I'm not sure. I've seen it 3 times: on 2017-02-18 (with 4.4.11), 2017-02-27 (with 4.4.11), and 2017-03-20 (with 4.4.12). The cron job runs daily. So I don't quite know what causes it. I've left it running, and can attach to it with GDB to answer questions, or anything. -- Happy hacking, ~ Luke Shumaker
Re: "unset var" pops var off variable stack instead of unsetting it
On 3/20/17 4:29 PM, Stephane Chazelas wrote:
>> I believe he means the behavior of `a=0; a=1 eval unset a', which Posix
>> implicitly requires affect the global scope and results in a being unset
>> when the statement completes.
> [...]
>
> See also:
>
> $ bash -c 'f() { unset a; echo "$a";}; a=1; a=2 f'
> 1
>
> already mentioned.
A distinction without a difference; the behavior is explicitly the same.
--
``The lyf so short, the craft so long to lerne.'' - Chaucer
``Ars longa, vita brevis'' - Hippocrates
Chet Ramey, UTech, CWRUc...@case.eduhttp://cnswww.cns.cwru.edu/~chet/
Re: "unset var" pops var off variable stack instead of unsetting it
2017-03-20 14:47:17 -0400, Chet Ramey:
> On 3/20/17 2:30 PM, Eric Blake wrote:
> > On 03/17/2017 07:21 PM, Stephane Chazelas wrote:
> >
> >>> The problem is the non-obvious nature of unset's interaction with scope,
> >>
> >> the main problem to me is an unset command that doesn't unset.
> >>
> >> As shown in my original post, there's also a POSIX conformance
> >> issue.
> >
> > As POSIX has not yet specified 'local', any use of 'local' already
> > renders the script non-conformant, so it shouldn't matter what bash does
> > in that situation (although if POSIX is ever going to standardize
> > 'local', it requires some concerted effort to make all shells with
> > 'local' to settle on a lowest common denominator).
>
> I believe he means the behavior of `a=0; a=1 eval unset a', which Posix
> implicitly requires affect the global scope and results in a being unset
> when the statement completes.
[...]
See also:
$ bash -c 'f() { unset a; echo "$a";}; a=1; a=2 f'
1
already mentioned.
In any case, those are corner cases I'm not too worried about.
I'm more concerned about "unset var" not unsetting var in real
life cases when "local"/"typeset" is being used (regardless of
POSIX).
The other POSIX related concern I was also mentioning is the
fact that the work around implies using non-POSIX constructs,
the fact that libraries of POSIX functions can't be used in
bash/mksh/yash.
The particular case that affects me directly, is that
recommendations I'm giving online about POSIX compatible
constructs such as:
(unset IFS; set -f; cmd -- $var)
Which is (or at least I though should be) *the* Bourne idiom to
split, already mentioned several times are *wrong* for bash (or
mksh or yash) in the general case, as "unset" doesn't do what it
says on the can there.
Another case of "principle of least astonishment" not being
followed IMO.
--
Stephane
Re: Bash monopolizing or eating the RAM MEMORY
thank you On Mon, Mar 20, 2017 at 4:03 PM, Greg Wooledge wrote: > On Mon, Mar 20, 2017 at 03:54:37PM -0300, Noilson Caio wrote: > > thank you so much Mr. Wooledge. i guess that BUG is a strong word for > this > > case. i fully agree about "his is not a bash bug. It's a problem with > your > > approach.", actuality that's my preoccupation. can you help me to > > understand because 10^6 strings pull the trigger of "Argument list too > > long" and 10^7(n+1) don't ? i have afraid that a non-root user can > > compromise a linux box intentionally. the memory needs be eaten until > other > > threshold can break it. > > It's not a "compromise". Any user on a computer can run a program that > uses (or tries to use) a bunch of memory. You as the system admin can > set resource limits on the user's processes. This is outside the scope > of the bug-bash mailing list (try a Linux sys admin list). > > As far as "argument list too long", I believe you already posted a link > to Sven Mascheck's ARG_MAX web page. This is the best explanation of > the concept and the details. For those that may have missed it, > see http://www.in-ulm.de/~mascheck/various/argmax/ > > If you want to create 1 million (or 10 million) directories from a bash > script, you're going to have to call mkdir repeatedly. If this is a > problem, then I suggest you rewrite in a langauge that can call mkdir() > as a function without forking a whole process to do so. Pretty much > every language that isn't a shell should allow this. Pick your favorite. > -- Noilson Caio Teixeira de Araújo https://ncaio.wordpress.com https://br.linkedin.com/in/ncaio https://twitter.com/noilsoncaio https://jammer4.wordpress.com/ http://8bit.academy
Re: Bash monopolizing or eating the RAM MEMORY
On Mon, Mar 20, 2017 at 03:54:37PM -0300, Noilson Caio wrote: > thank you so much Mr. Wooledge. i guess that BUG is a strong word for this > case. i fully agree about "his is not a bash bug. It's a problem with your > approach.", actuality that's my preoccupation. can you help me to > understand because 10^6 strings pull the trigger of "Argument list too > long" and 10^7(n+1) don't ? i have afraid that a non-root user can > compromise a linux box intentionally. the memory needs be eaten until other > threshold can break it. It's not a "compromise". Any user on a computer can run a program that uses (or tries to use) a bunch of memory. You as the system admin can set resource limits on the user's processes. This is outside the scope of the bug-bash mailing list (try a Linux sys admin list). As far as "argument list too long", I believe you already posted a link to Sven Mascheck's ARG_MAX web page. This is the best explanation of the concept and the details. For those that may have missed it, see http://www.in-ulm.de/~mascheck/various/argmax/ If you want to create 1 million (or 10 million) directories from a bash script, you're going to have to call mkdir repeatedly. If this is a problem, then I suggest you rewrite in a langauge that can call mkdir() as a function without forking a whole process to do so. Pretty much every language that isn't a shell should allow this. Pick your favorite.
Re: Bash monopolizing or eating the RAM MEMORY
On 3/20/17 2:54 PM, Noilson Caio wrote: > i have afraid that a non-root user can > compromise a linux box intentionally. the memory needs be eaten until other > threshold can break it. This is why per-process resource limits exist. -- ``The lyf so short, the craft so long to lerne.'' - Chaucer ``Ars longa, vita brevis'' - Hippocrates Chet Ramey, UTech, CWRUc...@case.eduhttp://cnswww.cns.cwru.edu/~chet/
Re: Bash monopolizing or eating the RAM MEMORY
thank you so much Mr. Wooledge. i guess that BUG is a strong word for this
case. i fully agree about "his is not a bash bug. It's a problem with your
approach.", actuality that's my preoccupation. can you help me to
understand because 10^6 strings pull the trigger of "Argument list too
long" and 10^7(n+1) don't ? i have afraid that a non-root user can
compromise a linux box intentionally. the memory needs be eaten until other
threshold can break it.
Thank you again.
On Mon, Mar 20, 2017 at 1:50 PM, Greg Wooledge wrote:
> On Mon, Mar 20, 2017 at 12:17:39PM -0300, Noilson Caio wrote:
> > 1 - Using mkdir -p {0..9}/{0..9}/{0..9}/{0..9}/{0..9}/ - ( 5 levels ) No
> > problems
>
> 10 to the 5th power (100,000) strings generated. Sloppy, but viable on
> today's computers. You're relying on your operating system to allow
> an extraordinary large set of arguments to processes. I'm guessing
> Linux.
>
> > 2 - Using mkdir -p {0..9}/{0..9}/{0..9}/{0..9}/{0..9}/{0..9}/ - (6
> levels )
> > We have a problem - "Argument list too long".
>
> You have two problems. The first is that you are generating 10^6
> (1 million) strings in memory, all at once. The second is that you are
> attempting to pass all of these strings as arguments to a single mkdir
> process. Apparently even your system won't permit that.
>
> > 3 - Using mkdir -p {0..9}/{0..9}/{0..9}/{0..9}/{0..9}/{0..9}/{0..9}/ -
> (7
> > levels ) - Ops, we don't have more "Argument list too long" now we have
> "Cannot
> > allocate memory".
>
> 10 million strings, all at once. Each one is ~15 bytes (counting the NUL
> and slashes), so you're looking at something like 150 megabytes.
>
> This is not a bash bug. It's a problem with your approach. You wouldn't
> call it a bug in C, if you wrote a C program that tried to allocate 150
> megabytes of variables and got an "out of memory" as a result. The same
> applies to any other programming language.
>
> What you need to do is actually think about how big a chunk of memory
> (and argument list) you can handle in a single call to mkdir -p, and
> just do that many at once. Call mkdir multiple times, in order to get
> the full task done. Don't assume bash will handle that for you.
>
--
Noilson Caio Teixeira de Araújo
https://ncaio.wordpress.com
https://br.linkedin.com/in/ncaio
https://twitter.com/noilsoncaio
https://jammer4.wordpress.com/
http://8bit.academy
Re: "unset var" pops var off variable stack instead of unsetting it
On 3/20/17 2:30 PM, Eric Blake wrote: > On 03/17/2017 07:21 PM, Stephane Chazelas wrote: > >>> The problem is the non-obvious nature of unset's interaction with scope, >> >> the main problem to me is an unset command that doesn't unset. >> >> As shown in my original post, there's also a POSIX conformance >> issue. > > As POSIX has not yet specified 'local', any use of 'local' already > renders the script non-conformant, so it shouldn't matter what bash does > in that situation (although if POSIX is ever going to standardize > 'local', it requires some concerted effort to make all shells with > 'local' to settle on a lowest common denominator). I believe he means the behavior of `a=0; a=1 eval unset a', which Posix implicitly requires affect the global scope and results in a being unset when the statement completes. -- ``The lyf so short, the craft so long to lerne.'' - Chaucer ``Ars longa, vita brevis'' - Hippocrates Chet Ramey, UTech, CWRUc...@case.eduhttp://cnswww.cns.cwru.edu/~chet/ signature.asc Description: OpenPGP digital signature
Re: "unset var" pops var off variable stack instead of unsetting it
On 03/17/2017 07:21 PM, Stephane Chazelas wrote: >> The problem is the non-obvious nature of unset's interaction with scope, > > the main problem to me is an unset command that doesn't unset. > > As shown in my original post, there's also a POSIX conformance > issue. As POSIX has not yet specified 'local', any use of 'local' already renders the script non-conformant, so it shouldn't matter what bash does in that situation (although if POSIX is ever going to standardize 'local', it requires some concerted effort to make all shells with 'local' to settle on a lowest common denominator). -- Eric Blake eblake redhat com+1-919-301-3266 Libvirt virtualization library http://libvirt.org signature.asc Description: OpenPGP digital signature
Re: Bash monopolizing or eating the RAM MEMORY
On Mon, Mar 20, 2017 at 10:17 AM, Noilson Caio wrote: > Configuration Information [Automatically generated, do not change]: > Machine: x86_64 > OS: linux-gnu > Compiler: gcc > Compilation CFLAGS: -DPROGRAM='bash' -DCONF_HOSTTYPE='x86_64' > -DCONF_OSTYPE='linux-gnu' -DCONF_MACHTYPE='x86_64-redhat-linux-gnu' > -DCONF_VENDOR='redhat' -DLOCALEDIR='/usr/share/locale' -DPACKAGE='bash' > -DSHELL -DHAVE_CONFIG_H -I. -I. -I./include -I./lib -D_GNU_SOURCE > -DRECYCLES_PIDS -DDEFAULT_PATH_VALUE='/usr/local/bin:/usr/bin' -O2 -g > -pipe -Wall -Wp,-D_FORTIFY_SOURCE=2 -fexceptions -fstack-protector-strong > --param=ssp-buffer-size=4 -grecord-gcc-switches -m64 -mtune=generic > uname output: Linux SPFBL-POC-CENTOS-7 3.10.0-514.2.2.el7.x86_64 #1 SMP > Tue Dec 6 23:06:41 UTC 2016 x86_64 x86_64 x86_64 GNU/Linux > Machine Type: x86_64-redhat-linux-gnu > > Bash Version: 4.2 > Patch Level: 46 > Release Status: release > > Description: > Hello bash crew. > My name is Noilson Caio and i assume that there is something weird/strange > in bash. I'm working with huge folder structures and a lot of files every > day and the best way to describe the 'problem' is using a example. > > Example task: Build a folder structure using 0-9 at 2 levels > Something like that: > I understand your problem. But what do you want BASH to do about it? BASH is a generalized command processor. You have found that it is possible to give a "reasonable" command which exhausts your available memory. No offense intended, but _I_ would consider that a "user error". It is no different from what I saw on another forum for the R language. The user exhausted memory by trying to read in a CSV file, the entire thing at once, which was about 1 million lines (rows) containing hundreds of values per row. Solution: buy a more powerful computer with, say, about 512 GiB of memory. It'll only set you back about $2,000 U.S.; example (not recommendation): http://www.thinkmate.com/system/rax-xs4-1160v4. Honestly: you need to change your algorithm for processing your problem. In your example, perhaps a series of nested loops. -- "Irrigation of the land with seawater desalinated by fusion power is ancient. It's called 'rain'." -- Michael McClary, in alt.fusion Maranatha! <>< John McKown
Re: Bash monopolizing or eating the RAM MEMORY
On Mon, Mar 20, 2017 at 12:17:39PM -0300, Noilson Caio wrote:
> 1 - Using mkdir -p {0..9}/{0..9}/{0..9}/{0..9}/{0..9}/ - ( 5 levels ) No
> problems
10 to the 5th power (100,000) strings generated. Sloppy, but viable on
today's computers. You're relying on your operating system to allow
an extraordinary large set of arguments to processes. I'm guessing
Linux.
> 2 - Using mkdir -p {0..9}/{0..9}/{0..9}/{0..9}/{0..9}/{0..9}/ - (6 levels )
> We have a problem - "Argument list too long".
You have two problems. The first is that you are generating 10^6
(1 million) strings in memory, all at once. The second is that you are
attempting to pass all of these strings as arguments to a single mkdir
process. Apparently even your system won't permit that.
> 3 - Using mkdir -p {0..9}/{0..9}/{0..9}/{0..9}/{0..9}/{0..9}/{0..9}/ - (7
> levels ) - Ops, we don't have more "Argument list too long" now we have
> "Cannot
> allocate memory".
10 million strings, all at once. Each one is ~15 bytes (counting the NUL
and slashes), so you're looking at something like 150 megabytes.
This is not a bash bug. It's a problem with your approach. You wouldn't
call it a bug in C, if you wrote a C program that tried to allocate 150
megabytes of variables and got an "out of memory" as a result. The same
applies to any other programming language.
What you need to do is actually think about how big a chunk of memory
(and argument list) you can handle in a single call to mkdir -p, and
just do that many at once. Call mkdir multiple times, in order to get
the full task done. Don't assume bash will handle that for you.
Bash monopolizing or eating the RAM MEMORY
Configuration Information [Automatically generated, do not change]:
Machine: x86_64
OS: linux-gnu
Compiler: gcc
Compilation CFLAGS: -DPROGRAM='bash' -DCONF_HOSTTYPE='x86_64'
-DCONF_OSTYPE='linux-gnu' -DCONF_MACHTYPE='x86_64-redhat-linux-gnu'
-DCONF_VENDOR='redhat' -DLOCALEDIR='/usr/share/locale' -DPACKAGE='bash'
-DSHELL -DHAVE_CONFIG_H -I. -I. -I./include -I./lib -D_GNU_SOURCE
-DRECYCLES_PIDS -DDEFAULT_PATH_VALUE='/usr/local/bin:/usr/bin' -O2 -g
-pipe -Wall -Wp,-D_FORTIFY_SOURCE=2 -fexceptions -fstack-protector-strong
--param=ssp-buffer-size=4 -grecord-gcc-switches -m64 -mtune=generic
uname output: Linux SPFBL-POC-CENTOS-7 3.10.0-514.2.2.el7.x86_64 #1 SMP Tue
Dec 6 23:06:41 UTC 2016 x86_64 x86_64 x86_64 GNU/Linux
Machine Type: x86_64-redhat-linux-gnu
Bash Version: 4.2
Patch Level: 46
Release Status: release
Description:
Hello bash crew.
My name is Noilson Caio and i assume that there is something weird/strange
in bash. I'm working with huge folder structures and a lot of files every
day and the best way to describe the 'problem' is using a example.
Example task: Build a folder structure using 0-9 at 2 levels
Something like that:
.
|-- 0
| |-- 0
| |-- 1
| |-- 2
| |-- 3
| |-- 4
| |-- 5
| |-- 6
| |-- 7
| |-- 8
| `-- 9
|-- 1
| |-- 0
| |-- 1
| |-- 2
| |-- 3
| |-- 4
| |-- 5
| |-- 6
| |-- 7
| |-- 8
| `-- 9
|-- 2
| |-- 0
| |-- 1
| |-- 2
| |-- 3
| |-- 4
| |-- 5
| |-- 6
| |-- 7
| |-- 8
| `-- 9
|-- 3
| |-- 0
| |-- 1
| |-- 2
| |-- 3
| |-- 4
| |-- 5
| |-- 6
| |-- 7
| |-- 8
| `-- 9
|-- 4
| |-- 0
| |-- 1
| |-- 2
| |-- 3
| |-- 4
| |-- 5
| |-- 6
| |-- 7
| |-- 8
| `-- 9
|-- 5
| |-- 0
| |-- 1
| |-- 2
| |-- 3
| |-- 4
| |-- 5
| |-- 6
| |-- 7
| |-- 8
| `-- 9
|-- 6
| |-- 0
| |-- 1
| |-- 2
| |-- 3
| |-- 4
| |-- 5
| |-- 6
| |-- 7
| |-- 8
| `-- 9
|-- 7
| |-- 0
| |-- 1
| |-- 2
| |-- 3
| |-- 4
| |-- 5
| |-- 6
| |-- 7
| |-- 8
| `-- 9
|-- 8
| |-- 0
| |-- 1
| |-- 2
| |-- 3
| |-- 4
| |-- 5
| |-- 6
| |-- 7
| |-- 8
| `-- 9
`-- 9
|-- 0
|-- 1
|-- 2
|-- 3
|-- 4
|-- 5
|-- 6
|-- 7
|-- 8
`-- 9
110 directories, 0 files
For this kind of job i've been using 'curly braces' '{}' for almost 10
years. In response to the question: mkdir -p {0..9}/{0..9}/
Well, so far so good. But when i grow the arguments list (folder levels)
strange things happen =]. let me show examples and facts.
1 - Using mkdir -p {0..9}/{0..9}/{0..9}/{0..9}/{0..9}/ - ( 5 levels ) No
problems
2 - Using mkdir -p {0..9}/{0..9}/{0..9}/{0..9}/{0..9}/{0..9}/ - (6 levels )
We have a problem - "Argument list too long". Not really a problem for the
bash, it's a problem to system operator. I know that's a ARG_MAX limitation.
When this happen, the operator fixed it with other tools/split/ways. Don't
make sense you do increase more arguments in this task, but let's go;
3 - Using mkdir -p {0..9}/{0..9}/{0..9}/{0..9}/{0..9}/{0..9}/{0..9}/ - (7
levels ) - Ops, we don't have more "Argument list too long" now we have "Cannot
allocate memory".
Strace sample:
access("/usr/bin/mkdir", X_OK) = 0 stat("/usr/bin/mkdir",
{st_mode=S_IFREG|0755, st_size=39680, ...}) = 0 geteuid() = 0 getegid() = 0
getuid() = 0 getgid() = 0 access("/usr/bin/mkdir", R_OK) = 0
stat("/usr/bin/mkdir", {st_mode=S_IFREG|0755, st_size=39680, ...}) = 0
stat("/usr/bin/mkdir", {st_mode=S_IFREG|0755, st_size=39680, ...}) = 0
geteuid() = 0 getegid() = 0 getuid() = 0 getgid() = 0
access("/usr/bin/mkdir", X_OK) = 0 stat("/usr/bin/mkdir",
{st_mode=S_IFREG|0755, st_size=39680, ...}) = 0 geteuid() = 0 getegid() = 0
getuid() = 0 getgid() = 0 access("/usr/bin/mkdir", R_OK) = 0
rt_sigprocmask(SIG_BLOCK, [INT CHLD], [], 8) = 0 clone(child_stack=0,
flags=CLONE_CHILD_CLEARTID|CLONE_CHILD_SETTID|SIGCHLD,
child_tidptr=0x7fc12637e9d0) = -1 ENOMEM (Cannot allocate memory) write(2,
"-bash: fork: Cannot allocate mem"..., 36) = 36
Basically all RAM MEMORY it was eaten. after that, bash cannot will be able
to create a new 'fork'
4 - Using mkdir -p {0..9}/{0..9}/{0..9}/{0..9}/{0..9}/{0..9}/{0..9}/{0..9}/
- ( 8 levels or more ). Well, in this case all ram memory and swap (if
there is) will be consumed. And only stop when kernel oom score send a SIG
to kill the bash process. Exhaustive brk() calls. Maybe the assumed limit
is virtual memory that for default is unlimited
PS:
Maximum RAM tested: 16 GB
Other bash version tested:
Configuration Information [Automatically generated, do not change]:
Machine: i586
OS: linux-gnu
Compiler: gcc
Compilation CFLAGS: -DPROGRAM='bash' -DCONF_HOSTTYPE='i586'
-DCONF_OSTYPE='linu
Re: "unset var" pops var off variable stack instead of unsetting it
It seems to me this matter is specific to the IFS variable (and possibly few others like CDPATH). Unsetting IFS restores the default field splitting behavior, but unsetting PATH does not restore the default command search path. As Peter suggests, you can locally re-define any variables you want and that should work in any situation. -- Yuki
Re: Odd LINENO behaviour in 'eval'
On 3/19/17 10:58 PM, Martijn Dekker wrote:
> Test script:
> printf "$LINENO "
> printf "$LINENO "
> eval 'printf "$LINENO "
> printf "$LINENO "
> printf "$LINENO " '
> printf "$LINENO\n"
>
> Output on various shells:
> bash: 1 2 5 6 7 6 (?!)
> {m,l,pd}ksh: 1 2 0 0 0 6 (?)
> AT&T ksh88: 1 2 3 3 3 6
> AT&T ksh93: 1 2 1 2 3 6
> FreeBSD sh: 1 2 1 2 3 6
> dash: 1 2 1 2 3 6
> yash: 1 2 1 2 3 6
> zsh (native): 1 2 1 2 3 6
> zsh (sh): 1 2 3 3 3 6 (like ksh88)
>
> It is unclear to me why bash starts counting the 'eval' lines at 5. Is
> this a bug?
Line 5 is the line that gets assigned to the `eval' command because that
is the current line when the complete command is parsed. You don't
really know the current line when a simple command starts to get parsed
(well, you do, but you don't really know when the simple command gets
started when you're dealing with bison).
Bash used to reset the line number to 1 when executing an `eval', but I
changed that back before bash-3.0 due to user requests.
--
``The lyf so short, the craft so long to lerne.'' - Chaucer
``Ars longa, vita brevis'' - Hippocrates
Chet Ramey, UTech, CWRUc...@case.eduhttp://cnswww.cns.cwru.edu/~chet/
Re: "unset var" pops var off variable stack instead of unsetting it
2017-03-20 08:04:33 -0400, Greg Wooledge: [...] > > Credits to Dan Douglas > > (https://www.mail-archive.com/miros-mksh@mirbsd.org/msg00707.html) > > for finding the bug. He did find a use for it though (get the > > value of a variable from the caller's scope). > > Isn't this exactly the same as the "upvar" trick that Freddy Vulto > discovered? http://www.fvue.nl/wiki/Bash:_Passing_variables_by_reference Hi Greg, Yes, I hadn't realised initially that the issue had already been discussed before (and not fixed). You'll find that that "upvar" and the link above has since been mentioned in this thread (see also https://www.mail-archive.com/bug-bash@gnu.org/msg19445.html) At this point, I have little hope that bash will be fixed. But mksh/oksh and yash still might. -- Stephane
Re: "unset var" pops var off variable stack instead of unsetting it
On Fri, Mar 17, 2017 at 09:51:34PM +, Stephane Chazelas wrote: > Then, the "unset", instead of unsetting IFS, actually pops a > layer off the stack. > Credits to Dan Douglas > (https://www.mail-archive.com/miros-mksh@mirbsd.org/msg00707.html) > for finding the bug. He did find a use for it though (get the > value of a variable from the caller's scope). Isn't this exactly the same as the "upvar" trick that Freddy Vulto discovered? http://www.fvue.nl/wiki/Bash:_Passing_variables_by_reference
Re: "unset var" pops var off variable stack instead of unsetting it
2017-03-20 12:30:09 +0900, 渡邊裕貴:
> It seems to me this matter is specific to the IFS variable (and possibly
> few others like CDPATH). Unsetting IFS restores the default field splitting
> behavior, but unsetting PATH does not restore the default command search
> path. As Peter suggests, you can locally re-define any variables you
> want and that should work in any situation.
[...]
Hi Yuki,
you seem to be concurring with me that unset is broken and that
the work around is to not use it.
Note that unsetting PATH generally *does* restore a default
command search path. However, on many systems, not everything
agrees on the default search path (for instance on my Debian
system, for execvp(), it's ":/bin:/usr/bin" (yes, current
directory first!), for bash and dash it seems to be only the
current directory (as if PATH was set to the empty string), for
yash it seems it's nothing, for mksh "/usr/bin:/bin", for ksh93
"/bin:/usr/bin"... behaviour left "implementation defined" by
POSIX) so unsetting PATH is not useful.
Now, there are many reasons one may want to use unset.
For instance, unsetting LC_* restores LANG, one may want to
unset LD_LIBRARY_PATH, GCONV_PATH, LOCPATH, PERL5LIB,
PYTHON_PATH... for security reasons or to get a consistent
behaviour. In POSIX sh language, unsetting a variable is the
only way to unexport it. Same for changing the type of a
variable to scalar in bash without declaring it local.
zsh/yash/mksh have "typeset -g" for that, but in bash typeset -g
affects the variable in the global scope instead of preventing
the restricting the scope in other shells.
unset is also commonly used to make sure variables /have a
default value of / like in things like:
rmv() (
unset OPTIND force interactive verbose
while getopts :ivf o; do
(f) force=1;;
...
esac
shift "$((OPTIND - 1))"
exec rm ... ${force+"-f"} "$@"
)
Replacing the "unset force" with "force=" (and use
${force:+"-f"}) to work around the unset bug would not be enough
with bash/mksh as $force might have been defined as "integer" in
a parent scope. So one would need to use "typeset force=", or
just "typeset foo" which declares it with an value,
but that would make it no longer standard sh code (so that
function can no longer be used in sh scripts).
--
Stephane
