Re: [03/10] genksyms: generate lexer and parser during build instead of shipping

2018-03-30 Thread Andrei Vagin 
On Sat, Mar 31, 2018 at 11:20:22AM +0900, Masahiro Yamada wrote:
> 2018-03-31 7:21 GMT+09:00 Andrei Vagin :
> > On Fri, Mar 30, 2018 at 10:40:22AM -0700, Andrei Vagin wrote:
> >> On Fri, Mar 23, 2018 at 10:04:32PM +0900, Masahiro Yamada wrote:
> >> > Now that the kernel build supports flex and bison, remove the _shipped
> >> > files and generate them during the build instead.
> >> >
> >> > There are no more shipped lexer and parser, so I ripped off the rules
> >> > in scripts/Malefile.lib that were used for REGENERATE_PARSERS.
> >> >
> >> > The genksyms parser has ambiguous grammar, which would emit warnings:
> >> >
> >> >  scripts/genksyms/parse.y: warning: 9 shift/reduce conflicts 
> >> > [-Wconflicts-sr]
> >> >  scripts/genksyms/parse.y: warning: 5 reduce/reduce conflicts 
> >> > [-Wconflicts-rr]
> >> >
> >> > They are normally suppressed, but displayed when W=1 is given.
> >> >
> >> > Signed-off-by: Masahiro Yamada 
> >> > ---
> >> >
> >> >  scripts/Makefile.lib |   24 +-
> >> >  scripts/genksyms/Makefile|   23 +
> >> >  scripts/genksyms/lex.lex.c_shipped   | 2291 
> >> > 
> >> >  scripts/genksyms/parse.tab.c_shipped | 2394 
> >> > --
> >> >  scripts/genksyms/parse.tab.h_shipped |  119 --
> >> >  5 files changed, 26 insertions(+), 4825 deletions(-)
> >> >  delete mode 100644 scripts/genksyms/lex.lex.c_shipped
> >> >  delete mode 100644 scripts/genksyms/parse.tab.c_shipped
> >> >  delete mode 100644 scripts/genksyms/parse.tab.h_shipped
> >> >
> >> > diff --git a/scripts/Makefile.lib b/scripts/Makefile.lib
> >> > index 2fde810..b7d2c97 100644
> >> > --- a/scripts/Makefile.lib
> >> > +++ b/scripts/Makefile.lib
> >> > @@ -183,14 +183,8 @@ endef
> >> >  quiet_cmd_flex = LEX $@
> >> >cmd_flex = $(LEX) -o$@ -L $<
> >> >
> >> > -ifdef REGENERATE_PARSERS
> >> > -.PRECIOUS: $(src)/%.lex.c_shipped
> >> > -$(src)/%.lex.c_shipped: $(src)/%.l
> >> > -   $(call cmd,flex)
> >> > -endif
> >> > -
> >> >  .PRECIOUS: $(obj)/%.lex.c
> >> > -$(filter %.lex.c,$(targets)): $(obj)/%.lex.c: $(src)/%.l FORCE
> >> > +$(obj)/%.lex.c: $(src)/%.l FORCE
> >> > $(call if_changed,flex)
> >> >
> >> >  # YACC
> >> > @@ -198,27 +192,15 @@ $(filter %.lex.c,$(targets)): $(obj)/%.lex.c: 
> >> > $(src)/%.l FORCE
> >> >  quiet_cmd_bison = YACC$@
> >> >cmd_bison = $(YACC) -o$@ -t -l $<
> >> >
> >> > -ifdef REGENERATE_PARSERS
> >> > -.PRECIOUS: $(src)/%.tab.c_shipped
> >> > -$(src)/%.tab.c_shipped: $(src)/%.y
> >> > -   $(call cmd,bison)
> >> > -endif
> >> > -
> >> >  .PRECIOUS: $(obj)/%.tab.c
> >> > -$(filter %.tab.c,$(targets)): $(obj)/%.tab.c: $(src)/%.y FORCE
> >> > +$(obj)/%.tab.c: $(src)/%.y FORCE
> >> > $(call if_changed,bison)
> >> >
> >> >  quiet_cmd_bison_h = YACC$@
> >> >cmd_bison_h = bison -o/dev/null --defines=$@ -t -l $<
> >> >
> >> > -ifdef REGENERATE_PARSERS
> >> > -.PRECIOUS: $(src)/%.tab.h_shipped
> >> > -$(src)/%.tab.h_shipped: $(src)/%.y
> >> > -   $(call cmd,bison_h)
> >> > -endif
> >> > -
> >> >  .PRECIOUS: $(obj)/%.tab.h
> >> > -$(filter %.tab.h,$(targets)): $(obj)/%.tab.h: $(src)/%.y FORCE
> >> > +$(obj)/%.tab.h: $(src)/%.y FORCE
> >> > $(call if_changed,bison_h)
> >> >
> >> >  # Shipped files
> >> > diff --git a/scripts/genksyms/Makefile b/scripts/genksyms/Makefile
> >> > index 0ccac51..f4749e8 100644
> >> > --- a/scripts/genksyms/Makefile
> >> > +++ b/scripts/genksyms/Makefile
> >> > @@ -5,9 +5,32 @@ always := $(hostprogs-y)
> >> >
> >> >  genksyms-objs  := genksyms.o parse.tab.o lex.lex.o
> >> >
> >> > +# FIXME: fix the ambiguous grammar in parse.y and delete this hack
> >> > +#
> >> > +# Suppress shift/reduce, reduce/reduce conflicts warnings
> >> > +# unless W=1 is specified.
> >> > +ifeq ($(findstring 1,$(KBUILD_ENABLE_EXTRA_GCC_CHECKS)),)
> >> > +SUPPRESS_BISON_WARNING := 2>/dev/null
> >>
> >> We have a robot which runs CRIU tests on linux-next.
> >> Yesterday it failed with this error:
> >>
> >>   HOSTCC  scripts/genksyms/genksyms.o
> >> make[2]: *** [scripts/genksyms/parse.tab.c] Error 127
> >>
> >> cripts/genksyms/Makefile:20: recipe for target 
> >> 'scripts/genksyms/parse.tab.c' failed
> >> scripts/Makefile.build:559: recipe for target 'scripts/genksyms' failed
> >> Makefile:1073: recipe for target 'scripts' failed
> >> make[1]: *** [scripts/genksyms] Error 2
> >> make: *** [scripts] Error 2
> >> make: *** Waiting for unfinished jobs
> >>
> >> https://travis-ci.org/avagin/linux/jobs/360056903
> >>
> >> From this output, it is very hard to understand what was going wrong.
> >
> >
> > The reason was that bison and fles were not installed, but I think the
> > error message should be more clear.
> >
> >>
> >> Thanks,
> >> Andrei
> >>
> 
> Thanks for the report.
> 
> 
> OK, I will apply the fix-up attached below.
> 
> If bison is not installed, it will fail with clear message.

Thank you!

> 
>   HOSTCC

Re: [03/10] genksyms: generate lexer and parser during build instead of shipping

2018-03-30 Thread Andrei Vagin 
On Sat, Mar 31, 2018 at 11:20:22AM +0900, Masahiro Yamada wrote:
> 2018-03-31 7:21 GMT+09:00 Andrei Vagin :
> > On Fri, Mar 30, 2018 at 10:40:22AM -0700, Andrei Vagin wrote:
> >> On Fri, Mar 23, 2018 at 10:04:32PM +0900, Masahiro Yamada wrote:
> >> > Now that the kernel build supports flex and bison, remove the _shipped
> >> > files and generate them during the build instead.
> >> >
> >> > There are no more shipped lexer and parser, so I ripped off the rules
> >> > in scripts/Malefile.lib that were used for REGENERATE_PARSERS.
> >> >
> >> > The genksyms parser has ambiguous grammar, which would emit warnings:
> >> >
> >> >  scripts/genksyms/parse.y: warning: 9 shift/reduce conflicts 
> >> > [-Wconflicts-sr]
> >> >  scripts/genksyms/parse.y: warning: 5 reduce/reduce conflicts 
> >> > [-Wconflicts-rr]
> >> >
> >> > They are normally suppressed, but displayed when W=1 is given.
> >> >
> >> > Signed-off-by: Masahiro Yamada 
> >> > ---
> >> >
> >> >  scripts/Makefile.lib |   24 +-
> >> >  scripts/genksyms/Makefile|   23 +
> >> >  scripts/genksyms/lex.lex.c_shipped   | 2291 
> >> > 
> >> >  scripts/genksyms/parse.tab.c_shipped | 2394 
> >> > --
> >> >  scripts/genksyms/parse.tab.h_shipped |  119 --
> >> >  5 files changed, 26 insertions(+), 4825 deletions(-)
> >> >  delete mode 100644 scripts/genksyms/lex.lex.c_shipped
> >> >  delete mode 100644 scripts/genksyms/parse.tab.c_shipped
> >> >  delete mode 100644 scripts/genksyms/parse.tab.h_shipped
> >> >
> >> > diff --git a/scripts/Makefile.lib b/scripts/Makefile.lib
> >> > index 2fde810..b7d2c97 100644
> >> > --- a/scripts/Makefile.lib
> >> > +++ b/scripts/Makefile.lib
> >> > @@ -183,14 +183,8 @@ endef
> >> >  quiet_cmd_flex = LEX $@
> >> >cmd_flex = $(LEX) -o$@ -L $<
> >> >
> >> > -ifdef REGENERATE_PARSERS
> >> > -.PRECIOUS: $(src)/%.lex.c_shipped
> >> > -$(src)/%.lex.c_shipped: $(src)/%.l
> >> > -   $(call cmd,flex)
> >> > -endif
> >> > -
> >> >  .PRECIOUS: $(obj)/%.lex.c
> >> > -$(filter %.lex.c,$(targets)): $(obj)/%.lex.c: $(src)/%.l FORCE
> >> > +$(obj)/%.lex.c: $(src)/%.l FORCE
> >> > $(call if_changed,flex)
> >> >
> >> >  # YACC
> >> > @@ -198,27 +192,15 @@ $(filter %.lex.c,$(targets)): $(obj)/%.lex.c: 
> >> > $(src)/%.l FORCE
> >> >  quiet_cmd_bison = YACC$@
> >> >cmd_bison = $(YACC) -o$@ -t -l $<
> >> >
> >> > -ifdef REGENERATE_PARSERS
> >> > -.PRECIOUS: $(src)/%.tab.c_shipped
> >> > -$(src)/%.tab.c_shipped: $(src)/%.y
> >> > -   $(call cmd,bison)
> >> > -endif
> >> > -
> >> >  .PRECIOUS: $(obj)/%.tab.c
> >> > -$(filter %.tab.c,$(targets)): $(obj)/%.tab.c: $(src)/%.y FORCE
> >> > +$(obj)/%.tab.c: $(src)/%.y FORCE
> >> > $(call if_changed,bison)
> >> >
> >> >  quiet_cmd_bison_h = YACC$@
> >> >cmd_bison_h = bison -o/dev/null --defines=$@ -t -l $<
> >> >
> >> > -ifdef REGENERATE_PARSERS
> >> > -.PRECIOUS: $(src)/%.tab.h_shipped
> >> > -$(src)/%.tab.h_shipped: $(src)/%.y
> >> > -   $(call cmd,bison_h)
> >> > -endif
> >> > -
> >> >  .PRECIOUS: $(obj)/%.tab.h
> >> > -$(filter %.tab.h,$(targets)): $(obj)/%.tab.h: $(src)/%.y FORCE
> >> > +$(obj)/%.tab.h: $(src)/%.y FORCE
> >> > $(call if_changed,bison_h)
> >> >
> >> >  # Shipped files
> >> > diff --git a/scripts/genksyms/Makefile b/scripts/genksyms/Makefile
> >> > index 0ccac51..f4749e8 100644
> >> > --- a/scripts/genksyms/Makefile
> >> > +++ b/scripts/genksyms/Makefile
> >> > @@ -5,9 +5,32 @@ always := $(hostprogs-y)
> >> >
> >> >  genksyms-objs  := genksyms.o parse.tab.o lex.lex.o
> >> >
> >> > +# FIXME: fix the ambiguous grammar in parse.y and delete this hack
> >> > +#
> >> > +# Suppress shift/reduce, reduce/reduce conflicts warnings
> >> > +# unless W=1 is specified.
> >> > +ifeq ($(findstring 1,$(KBUILD_ENABLE_EXTRA_GCC_CHECKS)),)
> >> > +SUPPRESS_BISON_WARNING := 2>/dev/null
> >>
> >> We have a robot which runs CRIU tests on linux-next.
> >> Yesterday it failed with this error:
> >>
> >>   HOSTCC  scripts/genksyms/genksyms.o
> >> make[2]: *** [scripts/genksyms/parse.tab.c] Error 127
> >>
> >> cripts/genksyms/Makefile:20: recipe for target 
> >> 'scripts/genksyms/parse.tab.c' failed
> >> scripts/Makefile.build:559: recipe for target 'scripts/genksyms' failed
> >> Makefile:1073: recipe for target 'scripts' failed
> >> make[1]: *** [scripts/genksyms] Error 2
> >> make: *** [scripts] Error 2
> >> make: *** Waiting for unfinished jobs
> >>
> >> https://travis-ci.org/avagin/linux/jobs/360056903
> >>
> >> From this output, it is very hard to understand what was going wrong.
> >
> >
> > The reason was that bison and fles were not installed, but I think the
> > error message should be more clear.
> >
> >>
> >> Thanks,
> >> Andrei
> >>
> 
> Thanks for the report.
> 
> 
> OK, I will apply the fix-up attached below.
> 
> If bison is not installed, it will fail with clear message.

Thank you!

> 
>   HOSTCC  scripts/genksyms/genksyms.o
> /bin/sh: 1: bison: not

Re: [03/10] genksyms: generate lexer and parser during build instead of shipping

2018-03-30 Thread Masahiro Yamada 
2018-03-31 7:21 GMT+09:00 Andrei Vagin :
> On Fri, Mar 30, 2018 at 10:40:22AM -0700, Andrei Vagin wrote:
>> On Fri, Mar 23, 2018 at 10:04:32PM +0900, Masahiro Yamada wrote:
>> > Now that the kernel build supports flex and bison, remove the _shipped
>> > files and generate them during the build instead.
>> >
>> > There are no more shipped lexer and parser, so I ripped off the rules
>> > in scripts/Malefile.lib that were used for REGENERATE_PARSERS.
>> >
>> > The genksyms parser has ambiguous grammar, which would emit warnings:
>> >
>> >  scripts/genksyms/parse.y: warning: 9 shift/reduce conflicts 
>> > [-Wconflicts-sr]
>> >  scripts/genksyms/parse.y: warning: 5 reduce/reduce conflicts 
>> > [-Wconflicts-rr]
>> >
>> > They are normally suppressed, but displayed when W=1 is given.
>> >
>> > Signed-off-by: Masahiro Yamada 
>> > ---
>> >
>> >  scripts/Makefile.lib |   24 +-
>> >  scripts/genksyms/Makefile|   23 +
>> >  scripts/genksyms/lex.lex.c_shipped   | 2291 
>> > 
>> >  scripts/genksyms/parse.tab.c_shipped | 2394 
>> > --
>> >  scripts/genksyms/parse.tab.h_shipped |  119 --
>> >  5 files changed, 26 insertions(+), 4825 deletions(-)
>> >  delete mode 100644 scripts/genksyms/lex.lex.c_shipped
>> >  delete mode 100644 scripts/genksyms/parse.tab.c_shipped
>> >  delete mode 100644 scripts/genksyms/parse.tab.h_shipped
>> >
>> > diff --git a/scripts/Makefile.lib b/scripts/Makefile.lib
>> > index 2fde810..b7d2c97 100644
>> > --- a/scripts/Makefile.lib
>> > +++ b/scripts/Makefile.lib
>> > @@ -183,14 +183,8 @@ endef
>> >  quiet_cmd_flex = LEX $@
>> >cmd_flex = $(LEX) -o$@ -L $<
>> >
>> > -ifdef REGENERATE_PARSERS
>> > -.PRECIOUS: $(src)/%.lex.c_shipped
>> > -$(src)/%.lex.c_shipped: $(src)/%.l
>> > -   $(call cmd,flex)
>> > -endif
>> > -
>> >  .PRECIOUS: $(obj)/%.lex.c
>> > -$(filter %.lex.c,$(targets)): $(obj)/%.lex.c: $(src)/%.l FORCE
>> > +$(obj)/%.lex.c: $(src)/%.l FORCE
>> > $(call if_changed,flex)
>> >
>> >  # YACC
>> > @@ -198,27 +192,15 @@ $(filter %.lex.c,$(targets)): $(obj)/%.lex.c: 
>> > $(src)/%.l FORCE
>> >  quiet_cmd_bison = YACC$@
>> >cmd_bison = $(YACC) -o$@ -t -l $<
>> >
>> > -ifdef REGENERATE_PARSERS
>> > -.PRECIOUS: $(src)/%.tab.c_shipped
>> > -$(src)/%.tab.c_shipped: $(src)/%.y
>> > -   $(call cmd,bison)
>> > -endif
>> > -
>> >  .PRECIOUS: $(obj)/%.tab.c
>> > -$(filter %.tab.c,$(targets)): $(obj)/%.tab.c: $(src)/%.y FORCE
>> > +$(obj)/%.tab.c: $(src)/%.y FORCE
>> > $(call if_changed,bison)
>> >
>> >  quiet_cmd_bison_h = YACC$@
>> >cmd_bison_h = bison -o/dev/null --defines=$@ -t -l $<
>> >
>> > -ifdef REGENERATE_PARSERS
>> > -.PRECIOUS: $(src)/%.tab.h_shipped
>> > -$(src)/%.tab.h_shipped: $(src)/%.y
>> > -   $(call cmd,bison_h)
>> > -endif
>> > -
>> >  .PRECIOUS: $(obj)/%.tab.h
>> > -$(filter %.tab.h,$(targets)): $(obj)/%.tab.h: $(src)/%.y FORCE
>> > +$(obj)/%.tab.h: $(src)/%.y FORCE
>> > $(call if_changed,bison_h)
>> >
>> >  # Shipped files
>> > diff --git a/scripts/genksyms/Makefile b/scripts/genksyms/Makefile
>> > index 0ccac51..f4749e8 100644
>> > --- a/scripts/genksyms/Makefile
>> > +++ b/scripts/genksyms/Makefile
>> > @@ -5,9 +5,32 @@ always := $(hostprogs-y)
>> >
>> >  genksyms-objs  := genksyms.o parse.tab.o lex.lex.o
>> >
>> > +# FIXME: fix the ambiguous grammar in parse.y and delete this hack
>> > +#
>> > +# Suppress shift/reduce, reduce/reduce conflicts warnings
>> > +# unless W=1 is specified.
>> > +ifeq ($(findstring 1,$(KBUILD_ENABLE_EXTRA_GCC_CHECKS)),)
>> > +SUPPRESS_BISON_WARNING := 2>/dev/null
>>
>> We have a robot which runs CRIU tests on linux-next.
>> Yesterday it failed with this error:
>>
>>   HOSTCC  scripts/genksyms/genksyms.o
>> make[2]: *** [scripts/genksyms/parse.tab.c] Error 127
>>
>> cripts/genksyms/Makefile:20: recipe for target 
>> 'scripts/genksyms/parse.tab.c' failed
>> scripts/Makefile.build:559: recipe for target 'scripts/genksyms' failed
>> Makefile:1073: recipe for target 'scripts' failed
>> make[1]: *** [scripts/genksyms] Error 2
>> make: *** [scripts] Error 2
>> make: *** Waiting for unfinished jobs
>>
>> https://travis-ci.org/avagin/linux/jobs/360056903
>>
>> From this output, it is very hard to understand what was going wrong.
>
>
> The reason was that bison and fles were not installed, but I think the
> error message should be more clear.
>
>>
>> Thanks,
>> Andrei
>>

Thanks for the report.


OK, I will apply the fix-up attached below.

If bison is not installed, it will fail with clear message.

  HOSTCC  scripts/genksyms/genksyms.o
/bin/sh: 1: bison: not found
make[2]: *** [scripts/genksyms/Makefile:18:
scripts/genksyms/parse.tab.c] Error 127
make[1]: *** [scripts/Makefile.build:559: scripts/genksyms] Error 2
make: *** [Makefile:1073: scripts] Error 2


BTW, without flex and bison, how did you build Kconfig?

Since commit

Re: [03/10] genksyms: generate lexer and parser during build instead of shipping

2018-03-30 Thread Masahiro Yamada 
2018-03-31 7:21 GMT+09:00 Andrei Vagin :
> On Fri, Mar 30, 2018 at 10:40:22AM -0700, Andrei Vagin wrote:
>> On Fri, Mar 23, 2018 at 10:04:32PM +0900, Masahiro Yamada wrote:
>> > Now that the kernel build supports flex and bison, remove the _shipped
>> > files and generate them during the build instead.
>> >
>> > There are no more shipped lexer and parser, so I ripped off the rules
>> > in scripts/Malefile.lib that were used for REGENERATE_PARSERS.
>> >
>> > The genksyms parser has ambiguous grammar, which would emit warnings:
>> >
>> >  scripts/genksyms/parse.y: warning: 9 shift/reduce conflicts 
>> > [-Wconflicts-sr]
>> >  scripts/genksyms/parse.y: warning: 5 reduce/reduce conflicts 
>> > [-Wconflicts-rr]
>> >
>> > They are normally suppressed, but displayed when W=1 is given.
>> >
>> > Signed-off-by: Masahiro Yamada 
>> > ---
>> >
>> >  scripts/Makefile.lib |   24 +-
>> >  scripts/genksyms/Makefile|   23 +
>> >  scripts/genksyms/lex.lex.c_shipped   | 2291 
>> > 
>> >  scripts/genksyms/parse.tab.c_shipped | 2394 
>> > --
>> >  scripts/genksyms/parse.tab.h_shipped |  119 --
>> >  5 files changed, 26 insertions(+), 4825 deletions(-)
>> >  delete mode 100644 scripts/genksyms/lex.lex.c_shipped
>> >  delete mode 100644 scripts/genksyms/parse.tab.c_shipped
>> >  delete mode 100644 scripts/genksyms/parse.tab.h_shipped
>> >
>> > diff --git a/scripts/Makefile.lib b/scripts/Makefile.lib
>> > index 2fde810..b7d2c97 100644
>> > --- a/scripts/Makefile.lib
>> > +++ b/scripts/Makefile.lib
>> > @@ -183,14 +183,8 @@ endef
>> >  quiet_cmd_flex = LEX $@
>> >cmd_flex = $(LEX) -o$@ -L $<
>> >
>> > -ifdef REGENERATE_PARSERS
>> > -.PRECIOUS: $(src)/%.lex.c_shipped
>> > -$(src)/%.lex.c_shipped: $(src)/%.l
>> > -   $(call cmd,flex)
>> > -endif
>> > -
>> >  .PRECIOUS: $(obj)/%.lex.c
>> > -$(filter %.lex.c,$(targets)): $(obj)/%.lex.c: $(src)/%.l FORCE
>> > +$(obj)/%.lex.c: $(src)/%.l FORCE
>> > $(call if_changed,flex)
>> >
>> >  # YACC
>> > @@ -198,27 +192,15 @@ $(filter %.lex.c,$(targets)): $(obj)/%.lex.c: 
>> > $(src)/%.l FORCE
>> >  quiet_cmd_bison = YACC$@
>> >cmd_bison = $(YACC) -o$@ -t -l $<
>> >
>> > -ifdef REGENERATE_PARSERS
>> > -.PRECIOUS: $(src)/%.tab.c_shipped
>> > -$(src)/%.tab.c_shipped: $(src)/%.y
>> > -   $(call cmd,bison)
>> > -endif
>> > -
>> >  .PRECIOUS: $(obj)/%.tab.c
>> > -$(filter %.tab.c,$(targets)): $(obj)/%.tab.c: $(src)/%.y FORCE
>> > +$(obj)/%.tab.c: $(src)/%.y FORCE
>> > $(call if_changed,bison)
>> >
>> >  quiet_cmd_bison_h = YACC$@
>> >cmd_bison_h = bison -o/dev/null --defines=$@ -t -l $<
>> >
>> > -ifdef REGENERATE_PARSERS
>> > -.PRECIOUS: $(src)/%.tab.h_shipped
>> > -$(src)/%.tab.h_shipped: $(src)/%.y
>> > -   $(call cmd,bison_h)
>> > -endif
>> > -
>> >  .PRECIOUS: $(obj)/%.tab.h
>> > -$(filter %.tab.h,$(targets)): $(obj)/%.tab.h: $(src)/%.y FORCE
>> > +$(obj)/%.tab.h: $(src)/%.y FORCE
>> > $(call if_changed,bison_h)
>> >
>> >  # Shipped files
>> > diff --git a/scripts/genksyms/Makefile b/scripts/genksyms/Makefile
>> > index 0ccac51..f4749e8 100644
>> > --- a/scripts/genksyms/Makefile
>> > +++ b/scripts/genksyms/Makefile
>> > @@ -5,9 +5,32 @@ always := $(hostprogs-y)
>> >
>> >  genksyms-objs  := genksyms.o parse.tab.o lex.lex.o
>> >
>> > +# FIXME: fix the ambiguous grammar in parse.y and delete this hack
>> > +#
>> > +# Suppress shift/reduce, reduce/reduce conflicts warnings
>> > +# unless W=1 is specified.
>> > +ifeq ($(findstring 1,$(KBUILD_ENABLE_EXTRA_GCC_CHECKS)),)
>> > +SUPPRESS_BISON_WARNING := 2>/dev/null
>>
>> We have a robot which runs CRIU tests on linux-next.
>> Yesterday it failed with this error:
>>
>>   HOSTCC  scripts/genksyms/genksyms.o
>> make[2]: *** [scripts/genksyms/parse.tab.c] Error 127
>>
>> cripts/genksyms/Makefile:20: recipe for target 
>> 'scripts/genksyms/parse.tab.c' failed
>> scripts/Makefile.build:559: recipe for target 'scripts/genksyms' failed
>> Makefile:1073: recipe for target 'scripts' failed
>> make[1]: *** [scripts/genksyms] Error 2
>> make: *** [scripts] Error 2
>> make: *** Waiting for unfinished jobs
>>
>> https://travis-ci.org/avagin/linux/jobs/360056903
>>
>> From this output, it is very hard to understand what was going wrong.
>
>
> The reason was that bison and fles were not installed, but I think the
> error message should be more clear.
>
>>
>> Thanks,
>> Andrei
>>

Thanks for the report.


OK, I will apply the fix-up attached below.

If bison is not installed, it will fail with clear message.

  HOSTCC  scripts/genksyms/genksyms.o
/bin/sh: 1: bison: not found
make[2]: *** [scripts/genksyms/Makefile:18:
scripts/genksyms/parse.tab.c] Error 127
make[1]: *** [scripts/Makefile.build:559: scripts/genksyms] Error 2
make: *** [Makefile:1073: scripts] Error 2


BTW, without flex and bison, how did you build Kconfig?

Since commit 29c833061c1d8c2d1d23a62e7061561eadd76cdb,
Kconfig requires flex and bison,

[PATCH 03/10] genksyms: generate lexer and parser during build instead of shipping

2018-03-23 Thread Masahiro Yamada 
Now that the kernel build supports flex and bison, remove the _shipped
files and generate them during the build instead.

There are no more shipped lexer and parser, so I ripped off the rules
in scripts/Malefile.lib that were used for REGENERATE_PARSERS.

The genksyms parser has ambiguous grammar, which would emit warnings:

 scripts/genksyms/parse.y: warning: 9 shift/reduce conflicts [-Wconflicts-sr]
 scripts/genksyms/parse.y: warning: 5 reduce/reduce conflicts [-Wconflicts-rr]

They are normally suppressed, but displayed when W=1 is given.

Signed-off-by: Masahiro Yamada 
---

 scripts/Makefile.lib |   24 +-
 scripts/genksyms/Makefile|   23 +
 scripts/genksyms/lex.lex.c_shipped   | 2291 
 scripts/genksyms/parse.tab.c_shipped | 2394 --
 scripts/genksyms/parse.tab.h_shipped |  119 --
 5 files changed, 26 insertions(+), 4825 deletions(-)
 delete mode 100644 scripts/genksyms/lex.lex.c_shipped
 delete mode 100644 scripts/genksyms/parse.tab.c_shipped
 delete mode 100644 scripts/genksyms/parse.tab.h_shipped

diff --git a/scripts/Makefile.lib b/scripts/Makefile.lib
index 2fde810..b7d2c97 100644
--- a/scripts/Makefile.lib
+++ b/scripts/Makefile.lib
@@ -183,14 +183,8 @@ endef
 quiet_cmd_flex = LEX $@
   cmd_flex = $(LEX) -o$@ -L $<
 
-ifdef REGENERATE_PARSERS
-.PRECIOUS: $(src)/%.lex.c_shipped
-$(src)/%.lex.c_shipped: $(src)/%.l
-   $(call cmd,flex)
-endif
-
 .PRECIOUS: $(obj)/%.lex.c
-$(filter %.lex.c,$(targets)): $(obj)/%.lex.c: $(src)/%.l FORCE
+$(obj)/%.lex.c: $(src)/%.l FORCE
$(call if_changed,flex)
 
 # YACC
@@ -198,27 +192,15 @@ $(filter %.lex.c,$(targets)): $(obj)/%.lex.c: $(src)/%.l 
FORCE
 quiet_cmd_bison = YACC$@
   cmd_bison = $(YACC) -o$@ -t -l $<
 
-ifdef REGENERATE_PARSERS
-.PRECIOUS: $(src)/%.tab.c_shipped
-$(src)/%.tab.c_shipped: $(src)/%.y
-   $(call cmd,bison)
-endif
-
 .PRECIOUS: $(obj)/%.tab.c
-$(filter %.tab.c,$(targets)): $(obj)/%.tab.c: $(src)/%.y FORCE
+$(obj)/%.tab.c: $(src)/%.y FORCE
$(call if_changed,bison)
 
 quiet_cmd_bison_h = YACC$@
   cmd_bison_h = bison -o/dev/null --defines=$@ -t -l $<
 
-ifdef REGENERATE_PARSERS
-.PRECIOUS: $(src)/%.tab.h_shipped
-$(src)/%.tab.h_shipped: $(src)/%.y
-   $(call cmd,bison_h)
-endif
-
 .PRECIOUS: $(obj)/%.tab.h
-$(filter %.tab.h,$(targets)): $(obj)/%.tab.h: $(src)/%.y FORCE
+$(obj)/%.tab.h: $(src)/%.y FORCE
$(call if_changed,bison_h)
 
 # Shipped files
diff --git a/scripts/genksyms/Makefile b/scripts/genksyms/Makefile
index 0ccac51..f4749e8 100644
--- a/scripts/genksyms/Makefile
+++ b/scripts/genksyms/Makefile
@@ -5,9 +5,32 @@ always := $(hostprogs-y)
 
 genksyms-objs  := genksyms.o parse.tab.o lex.lex.o
 
+# FIXME: fix the ambiguous grammar in parse.y and delete this hack
+#
+# Suppress shift/reduce, reduce/reduce conflicts warnings
+# unless W=1 is specified.
+ifeq ($(findstring 1,$(KBUILD_ENABLE_EXTRA_GCC_CHECKS)),)
+SUPPRESS_BISON_WARNING := 2>/dev/null
+endif
+
+quiet_cmd_bison_no_warn = $(quet_cmd_bison)
+  cmd_bison_no_warn = $(cmd_bison) $(SUPPRESS_BISON_WARNING)
+
+$(obj)/parse.tab.c: $(src)/parse.y FORCE
+   $(call if_changed,bison_no_warn)
+
+quiet_cmd_bison_h_no_warn = $(quet_cmd_bison_h)
+  cmd_bison_h_no_warn = $(cmd_bison_h) $(SUPPRESS_BISON_WARNING)
+
+$(obj)/parse.tab.h: $(src)/parse.y FORCE
+   $(call if_changed,bison_h_no_warn)
+# FIXME END
+
 # -I needed for generated C source (shipped source)
 HOSTCFLAGS_parse.tab.o := -I$(src)
 HOSTCFLAGS_lex.lex.o := -I$(src)
 
 # dependencies on generated files need to be listed explicitly
 $(obj)/lex.lex.o: $(obj)/parse.tab.h
+
+targets := lex.lex.c parse.tab.c parse.tab.h
diff --git a/scripts/genksyms/lex.lex.c_shipped 
b/scripts/genksyms/lex.lex.c_shipped
deleted file mode 100644
index ba2fda8..000
--- a/scripts/genksyms/lex.lex.c_shipped
+++ /dev/null
@@ -1,2291 +0,0 @@
-
-#line 3 "scripts/genksyms/lex.lex.c_shipped"
-
-#define  YY_INT_ALIGNED short int
-
-/* A lexical scanner generated by flex */
-
-#define FLEX_SCANNER
-#define YY_FLEX_MAJOR_VERSION 2
-#define YY_FLEX_MINOR_VERSION 5
-#define YY_FLEX_SUBMINOR_VERSION 35
-#if YY_FLEX_SUBMINOR_VERSION > 0
-#define FLEX_BETA
-#endif
-
-/* First, we deal with  platform-specific or compiler-specific issues. */
-
-/* begin standard C headers. */
-#include 
-#include 
-#include 
-#include 
-
-/* end standard C headers. */
-
-/* flex integer type definitions */
-
-#ifndef FLEXINT_H
-#define FLEXINT_H
-
-/* C99 systems have . Non-C99 systems may or may not. */
-
-#if defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L
-
-/* C99 says to define __STDC_LIMIT_MACROS before including stdint.h,
- * if you want the limit (max/min) macros for int types. 
- */
-#ifndef __STDC_LIMIT_MACROS
-#define __STDC_LIMIT_MACROS 1
-#endif
-
-#include 
-typedef int8_t flex_int8_t;
-typedef uint8_t flex_uint8_t;
-typedef int16_t flex_int16_t;
-typedef

[PATCH 03/10] genksyms: generate lexer and parser during build instead of shipping

2018-03-23 Thread Masahiro Yamada 
Now that the kernel build supports flex and bison, remove the _shipped
files and generate them during the build instead.

There are no more shipped lexer and parser, so I ripped off the rules
in scripts/Malefile.lib that were used for REGENERATE_PARSERS.

The genksyms parser has ambiguous grammar, which would emit warnings:

 scripts/genksyms/parse.y: warning: 9 shift/reduce conflicts [-Wconflicts-sr]
 scripts/genksyms/parse.y: warning: 5 reduce/reduce conflicts [-Wconflicts-rr]

They are normally suppressed, but displayed when W=1 is given.

Signed-off-by: Masahiro Yamada 
---

 scripts/Makefile.lib |   24 +-
 scripts/genksyms/Makefile|   23 +
 scripts/genksyms/lex.lex.c_shipped   | 2291 
 scripts/genksyms/parse.tab.c_shipped | 2394 --
 scripts/genksyms/parse.tab.h_shipped |  119 --
 5 files changed, 26 insertions(+), 4825 deletions(-)
 delete mode 100644 scripts/genksyms/lex.lex.c_shipped
 delete mode 100644 scripts/genksyms/parse.tab.c_shipped
 delete mode 100644 scripts/genksyms/parse.tab.h_shipped

diff --git a/scripts/Makefile.lib b/scripts/Makefile.lib
index 2fde810..b7d2c97 100644
--- a/scripts/Makefile.lib
+++ b/scripts/Makefile.lib
@@ -183,14 +183,8 @@ endef
 quiet_cmd_flex = LEX $@
   cmd_flex = $(LEX) -o$@ -L $<
 
-ifdef REGENERATE_PARSERS
-.PRECIOUS: $(src)/%.lex.c_shipped
-$(src)/%.lex.c_shipped: $(src)/%.l
-   $(call cmd,flex)
-endif
-
 .PRECIOUS: $(obj)/%.lex.c
-$(filter %.lex.c,$(targets)): $(obj)/%.lex.c: $(src)/%.l FORCE
+$(obj)/%.lex.c: $(src)/%.l FORCE
$(call if_changed,flex)
 
 # YACC
@@ -198,27 +192,15 @@ $(filter %.lex.c,$(targets)): $(obj)/%.lex.c: $(src)/%.l 
FORCE
 quiet_cmd_bison = YACC$@
   cmd_bison = $(YACC) -o$@ -t -l $<
 
-ifdef REGENERATE_PARSERS
-.PRECIOUS: $(src)/%.tab.c_shipped
-$(src)/%.tab.c_shipped: $(src)/%.y
-   $(call cmd,bison)
-endif
-
 .PRECIOUS: $(obj)/%.tab.c
-$(filter %.tab.c,$(targets)): $(obj)/%.tab.c: $(src)/%.y FORCE
+$(obj)/%.tab.c: $(src)/%.y FORCE
$(call if_changed,bison)
 
 quiet_cmd_bison_h = YACC$@
   cmd_bison_h = bison -o/dev/null --defines=$@ -t -l $<
 
-ifdef REGENERATE_PARSERS
-.PRECIOUS: $(src)/%.tab.h_shipped
-$(src)/%.tab.h_shipped: $(src)/%.y
-   $(call cmd,bison_h)
-endif
-
 .PRECIOUS: $(obj)/%.tab.h
-$(filter %.tab.h,$(targets)): $(obj)/%.tab.h: $(src)/%.y FORCE
+$(obj)/%.tab.h: $(src)/%.y FORCE
$(call if_changed,bison_h)
 
 # Shipped files
diff --git a/scripts/genksyms/Makefile b/scripts/genksyms/Makefile
index 0ccac51..f4749e8 100644
--- a/scripts/genksyms/Makefile
+++ b/scripts/genksyms/Makefile
@@ -5,9 +5,32 @@ always := $(hostprogs-y)
 
 genksyms-objs  := genksyms.o parse.tab.o lex.lex.o
 
+# FIXME: fix the ambiguous grammar in parse.y and delete this hack
+#
+# Suppress shift/reduce, reduce/reduce conflicts warnings
+# unless W=1 is specified.
+ifeq ($(findstring 1,$(KBUILD_ENABLE_EXTRA_GCC_CHECKS)),)
+SUPPRESS_BISON_WARNING := 2>/dev/null
+endif
+
+quiet_cmd_bison_no_warn = $(quet_cmd_bison)
+  cmd_bison_no_warn = $(cmd_bison) $(SUPPRESS_BISON_WARNING)
+
+$(obj)/parse.tab.c: $(src)/parse.y FORCE
+   $(call if_changed,bison_no_warn)
+
+quiet_cmd_bison_h_no_warn = $(quet_cmd_bison_h)
+  cmd_bison_h_no_warn = $(cmd_bison_h) $(SUPPRESS_BISON_WARNING)
+
+$(obj)/parse.tab.h: $(src)/parse.y FORCE
+   $(call if_changed,bison_h_no_warn)
+# FIXME END
+
 # -I needed for generated C source (shipped source)
 HOSTCFLAGS_parse.tab.o := -I$(src)
 HOSTCFLAGS_lex.lex.o := -I$(src)
 
 # dependencies on generated files need to be listed explicitly
 $(obj)/lex.lex.o: $(obj)/parse.tab.h
+
+targets := lex.lex.c parse.tab.c parse.tab.h
diff --git a/scripts/genksyms/lex.lex.c_shipped 
b/scripts/genksyms/lex.lex.c_shipped
deleted file mode 100644
index ba2fda8..000
--- a/scripts/genksyms/lex.lex.c_shipped
+++ /dev/null
@@ -1,2291 +0,0 @@
-
-#line 3 "scripts/genksyms/lex.lex.c_shipped"
-
-#define  YY_INT_ALIGNED short int
-
-/* A lexical scanner generated by flex */
-
-#define FLEX_SCANNER
-#define YY_FLEX_MAJOR_VERSION 2
-#define YY_FLEX_MINOR_VERSION 5
-#define YY_FLEX_SUBMINOR_VERSION 35
-#if YY_FLEX_SUBMINOR_VERSION > 0
-#define FLEX_BETA
-#endif
-
-/* First, we deal with  platform-specific or compiler-specific issues. */
-
-/* begin standard C headers. */
-#include 
-#include 
-#include 
-#include 
-
-/* end standard C headers. */
-
-/* flex integer type definitions */
-
-#ifndef FLEXINT_H
-#define FLEXINT_H
-
-/* C99 systems have . Non-C99 systems may or may not. */
-
-#if defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L
-
-/* C99 says to define __STDC_LIMIT_MACROS before including stdint.h,
- * if you want the limit (max/min) macros for int types. 
- */
-#ifndef __STDC_LIMIT_MACROS
-#define __STDC_LIMIT_MACROS 1
-#endif
-
-#include 
-typedef int8_t flex_int8_t;
-typedef uint8_t flex_uint8_t;
-typedef int16_t flex_int16_t;
-typedef uint16_t flex_uint16_t;
-typedef