[OE-core] [PATCH V2 07/11] pybootchartgui/parsing.py: fix error handling in meminfo parser
When matching fails, m.group(0) is invalid and can't be used in the
error message.
Signed-off-by: Patrick Ohly <patrick.o...@intel.com>
---
scripts/pybootchartgui/pybootchartgui/parsing.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/scripts/pybootchartgui/pybootchartgui/parsing.py
b/scripts/pybootchartgui/pybootchartgui/parsing.py
index af68435..48eb048 100644
--- a/scripts/pybootchartgui/pybootchartgui/parsing.py
+++ b/scripts/pybootchartgui/pybootchartgui/parsing.py
@@ -498,7 +498,7 @@ def _parse_proc_meminfo_log(file):
for line in lines:
match = meminfo_re.match(line)
if not match:
-raise ParseError("Invalid meminfo line \"%s\"" %
match.groups(0))
+raise ParseError("Invalid meminfo line \"%s\"" % line)
sample.add_value(match.group(1), int(match.group(2)))
if sample.valid():
--
2.1.4
--
___
Openembedded-core mailing list
Openembedded-core@lists.openembedded.org
http://lists.openembedded.org/mailman/listinfo/openembedded-core
[OE-core] [PATCH V2 05/11] pybootchartgui/draw.py: skip empty CPU and disk usage charts
The only real change is the addition of two if checks that skips the corresponding drawing code when there is no data. Signed-off-by: Patrick Ohly <patrick.o...@intel.com> --- scripts/pybootchartgui/pybootchartgui/draw.py | 98 ++- 1 file changed, 50 insertions(+), 48 deletions(-) diff --git a/scripts/pybootchartgui/pybootchartgui/draw.py b/scripts/pybootchartgui/pybootchartgui/draw.py index bddd804..ec5dd33 100644 --- a/scripts/pybootchartgui/pybootchartgui/draw.py +++ b/scripts/pybootchartgui/pybootchartgui/draw.py @@ -344,56 +344,58 @@ def render_charts(ctx, options, clip, trace, curr_y, w, h, sec_w): proc_tree = options.proc_tree(trace) # render bar legend - ctx.set_font_size(LEGEND_FONT_SIZE) - - draw_legend_box(ctx, "CPU (user+sys)", CPU_COLOR, off_x, curr_y+20, leg_s) - draw_legend_box(ctx, "I/O (wait)", IO_COLOR, off_x + 120, curr_y+20, leg_s) - - # render I/O wait - chart_rect = (off_x, curr_y+30, w, bar_h) - if clip_visible (clip, chart_rect): - draw_box_ticks (ctx, chart_rect, sec_w) - draw_annotations (ctx, proc_tree, trace.times, chart_rect) - draw_chart (ctx, IO_COLOR, True, chart_rect, \ - [(sample.time, sample.user + sample.sys + sample.io) for sample in trace.cpu_stats], \ - proc_tree, None) - # render CPU load - draw_chart (ctx, CPU_COLOR, True, chart_rect, \ - [(sample.time, sample.user + sample.sys) for sample in trace.cpu_stats], \ - proc_tree, None) - - curr_y = curr_y + 30 + bar_h + if trace.cpu_stats: + ctx.set_font_size(LEGEND_FONT_SIZE) + + draw_legend_box(ctx, "CPU (user+sys)", CPU_COLOR, off_x, curr_y+20, leg_s) + draw_legend_box(ctx, "I/O (wait)", IO_COLOR, off_x + 120, curr_y+20, leg_s) + + # render I/O wait + chart_rect = (off_x, curr_y+30, w, bar_h) + if clip_visible (clip, chart_rect): + draw_box_ticks (ctx, chart_rect, sec_w) + draw_annotations (ctx, proc_tree, trace.times, chart_rect) + draw_chart (ctx, IO_COLOR, True, chart_rect, \ + [(sample.time, sample.user + sample.sys + sample.io) for sample in trace.cpu_stats], \ + proc_tree, None) + # render CPU load + draw_chart (ctx, CPU_COLOR, True, chart_rect, \ + [(sample.time, sample.user + sample.sys) for sample in trace.cpu_stats], \ + proc_tree, None) + + curr_y = curr_y + 30 + bar_h # render second chart - draw_legend_line(ctx, "Disk throughput", DISK_TPUT_COLOR, off_x, curr_y+20, leg_s) - draw_legend_box(ctx, "Disk utilization", IO_COLOR, off_x + 120, curr_y+20, leg_s) - -# render I/O utilization - chart_rect = (off_x, curr_y+30, w, bar_h) - if clip_visible (clip, chart_rect): - draw_box_ticks (ctx, chart_rect, sec_w) - draw_annotations (ctx, proc_tree, trace.times, chart_rect) - draw_chart (ctx, IO_COLOR, True, chart_rect, \ - [(sample.time, sample.util) for sample in trace.disk_stats], \ - proc_tree, None) - - # render disk throughput - max_sample = max (trace.disk_stats, key = lambda s: s.tput) - if clip_visible (clip, chart_rect): - draw_chart (ctx, DISK_TPUT_COLOR, False, chart_rect, \ - [(sample.time, sample.tput) for sample in trace.disk_stats], \ - proc_tree, None) - - pos_x = off_x + ((max_sample.time - proc_tree.start_time) * w / proc_tree.duration) - - shift_x, shift_y = -20, 20 - if (pos_x < off_x + 245): - shift_x, shift_y = 5, 40 - - label = "%dMB/s" % round ((max_sample.tput) / 1024.0) - draw_text (ctx, label, DISK_TPUT_COLOR, pos_x + shift_x, curr_y + shift_y) - - curr_y = curr_y + 30 + bar_h + if trace.disk_stats: + draw_legend_line(ctx, "Disk throughput", DISK_TPUT_COLOR, off_x, curr_y+20, leg_s) + draw_legend_box(ctx, "Disk utilization", IO_COLOR, off_x + 120, curr_y+20, leg_s) + + # render I/O utilization + chart_rect = (off_x, curr_y+30, w, bar_h) + if clip_visible (clip, chart_rect): + draw_box_ticks (ctx, chart_rect, sec_w) + draw_annotations (ctx, proc_tree, trace.times, chart_rect) + draw_chart (ctx, IO_COLOR, True, chart_rect
[OE-core] [PATCH V2 01/11] buildstats: add system state sampling
/proc/[diskstats|meminfo|stat] get sampled and written to the same
proc_.log files as during normal bootchat logging. This will
allow rendering the CPU, disk and memory usage charts.
Right now sampling happens once a second, triggered by the heartbeat
event.That produces quite a bit of data for long builds, which will be
addressed in a separate commit by storing the data in a more compact
form.
Signed-off-by: Patrick Ohly <patrick.o...@intel.com>
---
meta/classes/buildstats.bbclass | 24 +
meta/lib/buildstats.py | 47 +
2 files changed, 71 insertions(+)
create mode 100644 meta/lib/buildstats.py
diff --git a/meta/classes/buildstats.bbclass b/meta/classes/buildstats.bbclass
index 57ecc8f..9c0c37d 100644
--- a/meta/classes/buildstats.bbclass
+++ b/meta/classes/buildstats.bbclass
@@ -188,3 +188,27 @@ python run_buildstats () {
addhandler run_buildstats
run_buildstats[eventmask] = "bb.event.BuildStarted bb.event.BuildCompleted
bb.build.TaskStarted bb.build.TaskSucceeded bb.build.TaskFailed"
+python runqueue_stats () {
+import buildstats
+from bb import event, runqueue
+# We should not record any samples before the first task has started,
+# because that's the first activity shown in the process chart.
+# Besides, at that point we are sure that the build variables
+# are available that we need to find the output directory.
+# The persistent SystemStats is stored in the datastore and
+# closed when the build is done.
+system_stats = d.getVar('_buildstats_system_stats', True)
+if not system_stats and isinstance(e, (bb.runqueue.sceneQueueTaskStarted,
bb.runqueue.runQueueTaskStarted)):
+system_stats = buildstats.SystemStats(d)
+d.setVar('_buildstats_system_stats', system_stats)
+if system_stats:
+# Ensure that we sample at important events.
+done = isinstance(e, bb.event.BuildCompleted)
+system_stats.sample(force=done)
+if done:
+system_stats.close()
+d.delVar('_buildstats_system_stats')
+}
+
+addhandler runqueue_stats
+runqueue_stats[eventmask] = "bb.runqueue.sceneQueueTaskStarted
bb.runqueue.runQueueTaskStarted bb.event.HeartbeatEvent bb.event.BuildCompleted"
diff --git a/meta/lib/buildstats.py b/meta/lib/buildstats.py
new file mode 100644
index 000..8ce4112
--- /dev/null
+++ b/meta/lib/buildstats.py
@@ -0,0 +1,47 @@
+# Implements system state sampling. Called by buildstats.bbclass.
+# Because it is a real Python module, it can hold persistent state,
+# like open log files and the time of the last sampling.
+
+import time
+
+class SystemStats:
+def __init__(self, d):
+bn = d.getVar('BUILDNAME', True)
+bsdir = os.path.join(d.getVar('BUILDSTATS_BASE', True), bn)
+bb.utils.mkdirhier(bsdir)
+
+self.proc_files = []
+for filename in ('diskstats', 'meminfo', 'stat'):
+# In practice, this class gets instantiated only once in
+# the bitbake cooker process. Therefore 'append' mode is
+# not strictly necessary, but using it makes the class
+# more robust should two processes ever write
+# concurrently.
+self.proc_files.append((filename,
+open(os.path.join(bsdir, 'proc_%s.log' %
filename), 'ab')))
+# Last time that we sampled data.
+self.last = 0
+# Minimum number of seconds between recording a sample. This
+# becames relevant when we get called very often while many
+# short tasks get started. Sampling during quiet periods
+# depends on the heartbeat event, which fires less often.
+self.min_seconds = 1
+
+def close(self):
+self.monitor_disk.close()
+for _, output, _ in self.proc_files:
+output.close()
+
+def sample(self, force):
+now = time.time()
+if (now - self.last > self.min_seconds) or force:
+for filename, output in self.proc_files:
+with open(os.path.join('/proc', filename), 'rb') as input:
+data = input.read()
+# Unbuffered raw write, less overhead and useful
+# in case that we end up with concurrent writes.
+os.write(output.fileno(),
+ ('%.0f\n' % now).encode('ascii') +
+ data +
+ b'\n')
+self.last = now
--
2.1.4
--
___
Openembedded-core mailing list
Openembedded-core@lists.openembedded.org
http://lists.openembedded.org/mailman/listinfo/openembedded-core
[OE-core] [PATCH V2 02/11] pybootchartgui/draw.py: allow moving process chart up and down
Substracting curr_y when determining the hight of the process chart is wrong because the height is independent of the position where the chart is about to be drawn. It happens to work at the moment because curr_y is always 10 when render_processes_chart() gets called. But it leads to a negative height when other charts are drawn above it, and then the grid gets drawn on top of those other charts. Substracting some constant is relevant because otherwise the box is slightly larger than the process bars. Not sure exactly where that comes from (text height?); leg_s seems a suitable constant and happens to be 10, so everything still gets rendered exactly as before. Signed-off-by: Patrick Ohly <patrick.o...@intel.com> --- scripts/pybootchartgui/pybootchartgui/draw.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/pybootchartgui/pybootchartgui/draw.py b/scripts/pybootchartgui/pybootchartgui/draw.py index 8c574be..2b5907b 100644 --- a/scripts/pybootchartgui/pybootchartgui/draw.py +++ b/scripts/pybootchartgui/pybootchartgui/draw.py @@ -415,7 +415,7 @@ def render_charts(ctx, options, clip, trace, curr_y, w, h, sec_w): return curr_y def render_processes_chart(ctx, options, trace, curr_y, w, h, sec_w): -chart_rect = [off_x, curr_y+header_h, w, h - 2 * off_y - (curr_y+header_h) + proc_h] +chart_rect = [off_x, curr_y+header_h, w, h - 2 * off_y - header_h - leg_s + proc_h] draw_legend_box (ctx, "Configure", \ TASK_COLOR_CONFIGURE, off_x , curr_y + 45, leg_s) -- 2.1.4 -- ___ Openembedded-core mailing list Openembedded-core@lists.openembedded.org http://lists.openembedded.org/mailman/listinfo/openembedded-core
[OE-core] [PATCH V2 00/11] system statistics sampling
buildstats.bbclass gets extended so that disk bandwidth and capacity,
CPU and memory usage get sampled at regular time intervals by
default. pybootchart had code for most of that data, which gets
restored and re-enabled. The disk capacity graph is new.
The original pybootchart just stored raw dumps of the special files
under /proc. bitbake builds run longer, so a more compact ("reduced")
format is used.
Logging disk capacity was mostly motivated by some work on enhancing
rm_work.bbclass behavior, but could also be useful for others ("how
much free space did my build need").
Changes in V2:
- SystemStats instance survives between event handler calls by
storing it in the corresponding datastore, instead of using
a global variable, and open files get closed properly when
the build ends. This should work better with memres bitbake (not
tested, though, because of YOCTO #10741).
- Sampling now already starts when setscene tasks start to run.
The following changes since commit 9f1fe76727e98e58fc9e46ea2b49cf5c0cb48e6c:
libpcap: Fix build when PACKAGECONFIG ipv6 is not enable (2016-11-23 11:02:33
+)
are available in the git repository at:
git://github.com/pohly/openembedded-core buildstats
https://github.com/pohly/openembedded-core/tree/buildstats
Patrick Ohly (11):
buildstats: add system state sampling
pybootchartgui/draw.py: allow moving process chart up and down
pybootchartgui/draw.py: fix drawing of samples not starting at zero
pybootchartgui: show system utilization
pybootchartgui/draw.py: skip empty CPU and disk usage charts
buildstats: record disk space usage
pybootchartgui/parsing.py: fix error handling in meminfo parser
pybootchartgui: render disk space usage
pybootchartgui: simplify drawing of memory usage
buildstats: reduce amount of data stored for system utilization
pybootchartgui: support reading reduced /proc logs
meta/classes/buildstats.bbclass | 24 +++
meta/lib/buildstats.py | 153 ++
scripts/pybootchartgui/pybootchartgui/draw.py| 192 ---
scripts/pybootchartgui/pybootchartgui/parsing.py | 122 +++---
scripts/pybootchartgui/pybootchartgui/samples.py | 27
5 files changed, 440 insertions(+), 78 deletions(-)
create mode 100644 meta/lib/buildstats.py
--
2.1.4
--
___
Openembedded-core mailing list
Openembedded-core@lists.openembedded.org
http://lists.openembedded.org/mailman/listinfo/openembedded-core
[OE-core] [PATCH V3] scripts/send-pull-request: Avoid multiple chain headers
When creating a patch set with cover letter using the send-pull-request script, both the "In-Reply-To" and "References" headers are appended twice in patch 2 and subsequent. That's because git-format-patch already inserted them and then git-send-email repeats that. Suppressing mail threading in git-send-email with --no-thread avoids the problem and is the right solution because it works regardless whether git-send-email is called once or twicee. Repeating these headers is a violation of RFC 2822 and can confuse mail programs. For example, Patchwork does not detect a patch series problem when there are these extra headers. [YOCTO #10718] Signed-off-by: Patrick Ohly <patrick.o...@intel.com> --- scripts/send-pull-request | 9 +++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/scripts/send-pull-request b/scripts/send-pull-request index 575549d..883deac 100755 --- a/scripts/send-pull-request +++ b/scripts/send-pull-request @@ -158,11 +158,16 @@ GIT_EXTRA_CC=$(for R in $EXTRA_CC; do echo -n "--cc='$R' "; done) unset IFS # Handoff to git-send-email. It will perform the send confirmation. +# Mail threading was already handled by git-format-patch in +# create-pull-request, so we must not allow git-send-email to +# add In-Reply-To and References headers again. PATCHES=$(echo $PDIR/*.patch) if [ $AUTO_CL -eq 1 ]; then # Send the cover letter to every recipient, both specified as well as # harvested. Then remove it from the patches list. - eval "git send-email $GIT_TO $GIT_CC $GIT_EXTRA_CC --confirm=always --no-chain-reply-to --suppress-cc=all $CL" + # --no-thread is redundant here (only sending a single message) and + # merely added for the sake of consistency. + eval "git send-email $GIT_TO $GIT_CC $GIT_EXTRA_CC --confirm=always --no-thread --suppress-cc=all $CL" if [ $? -eq 1 ]; then echo "ERROR: failed to send cover-letter with automatic recipients." exit 1 @@ -172,7 +177,7 @@ fi # Send the patch to the specified recipients and, if -c was specified, those git # finds in this specific patch. -eval "git send-email $GIT_TO $GIT_EXTRA_CC --confirm=always --no-chain-reply-to $GITSOBCC $PATCHES" +eval "git send-email $GIT_TO $GIT_EXTRA_CC --confirm=always --no-thread $GITSOBCC $PATCHES" if [ $? -eq 1 ]; then echo "ERROR: failed to send patches." exit 1 -- 2.1.4 -- ___ Openembedded-core mailing list Openembedded-core@lists.openembedded.org http://lists.openembedded.org/mailman/listinfo/openembedded-core
Re: [OE-core] [PATCH V2] scripts.send-pull-request: Avoid multiple chain headers
On Mon, 2016-11-28 at 16:35 -0600, Jose Lamego wrote: > > On 11/28/2016 03:34 PM, Patrick Ohly wrote: > > On Mon, 2016-11-28 at 14:28 -0600, Jose Lamego wrote: > >> Agree. Please provide feedback about below comments and I will submit a > >> v3 patch. > >> > >> On 11/28/2016 01:47 PM, Patrick Ohly wrote: > >>> On Mon, 2016-11-28 at 10:23 -0600, Jose Lamego wrote: > >> More than 1 "In-Reply-To" and "References" message headers are in > >> violation of rfc2822 [1] and may cause that some email-related > >> applications do not point to the appropriate root message in a > >> conversation/series. > > > > Fixing that makes sense. Just add it as reason and the "why" part is > > covered. > > > >>> And I don't understand why this proposed change has the described > >>> effect. Does changing the threading parameters change the output of "git > >>> send-email" and thus indirectly the mail headers of the following > >>> patches? > > > > The "how" part still isn't clear to me. Perhaps I'm just dumb, but would > > you bear with me and explain a bit more how changing the sending of the > > cover letter affects sending of the patches? I've tried out your proposed change with bash -x ../poky/scripts/send-pull-request --to=patrick.o...@gmx.de -p pull-11827 where pull-11827 is my recent bitbake submission. The resulting emails are still broken because that one line that you modify isn't event used. It's under "if [ $AUTO_CL -eq 1 ]" and I am not using the -a option that enables that behavior. Even when I use -a, the result is still broken. The root cause of the problem is that both create-pull-request and send-pull-request allow git to insert In-Reply-To headers. "git send-email --help" explicitly warns about that: It is up to the user to ensure that no In-Reply-To header already exists when git send-email is asked to add it (especially note that git format-patch can be configured to do the threading itself). Failure to do so may not produce the expected result in the recipient’s MUA. > What I'm doing > here is to include no reference to any root message at the first call, > then including a reference at the second call to the very first message > in the chain, which is either the cover letter or the patch #1. No, that doesn't work. Whether the first call uses --no-thread or --no-chain-reply-to has no effect whatsoever, because when "git send-email" only sends a single email, it doesn't add headers, and the second call was left unmodified in your patch. The right fix (tested successfully here) is to use --no-thread in the second call which sends the sequence of patches. I'll send my change for review separately. -- Best Regards, Patrick Ohly The content of this message is my personal opinion only and although I am an employee of Intel, the statements I make here in no way represent Intel's position on the issue, nor am I authorized to speak on behalf of Intel on this matter. -- ___ Openembedded-core mailing list Openembedded-core@lists.openembedded.org http://lists.openembedded.org/mailman/listinfo/openembedded-core
Re: [OE-core] [PATCH V2] scripts.send-pull-request: Avoid multiple chain headers
On Mon, 2016-11-28 at 14:28 -0600, Jose Lamego wrote: > Agree. Please provide feedback about below comments and I will submit a > v3 patch. > > On 11/28/2016 01:47 PM, Patrick Ohly wrote: > > On Mon, 2016-11-28 at 10:23 -0600, Jose Lamego wrote: > More than 1 "In-Reply-To" and "References" message headers are in > violation of rfc2822 [1] and may cause that some email-related > applications do not point to the appropriate root message in a > conversation/series. Fixing that makes sense. Just add it as reason and the "why" part is covered. > > And I don't understand why this proposed change has the described > > effect. Does changing the threading parameters change the output of "git > > send-email" and thus indirectly the mail headers of the following > > patches? The "how" part still isn't clear to me. Perhaps I'm just dumb, but would you bear with me and explain a bit more how changing the sending of the cover letter affects sending of the patches? As it isn't obvious, perhaps even add a comment to the script explaining it. -- Best Regards, Patrick Ohly The content of this message is my personal opinion only and although I am an employee of Intel, the statements I make here in no way represent Intel's position on the issue, nor am I authorized to speak on behalf of Intel on this matter. -- ___ Openembedded-core mailing list Openembedded-core@lists.openembedded.org http://lists.openembedded.org/mailman/listinfo/openembedded-core
Re: [OE-core] [PATCH V2] scripts.send-pull-request: Avoid multiple chain headers
On Mon, 2016-11-28 at 10:23 -0600, Jose Lamego wrote: > When creating a patch set with cover letter using the > send-pull-request script, both the "In-Reply-To" and "References" > headers are appended twice in patch 2 and subsequent. The "why" part is missing in the commit header. "Why" is appending those twice a problem? Is it a bug in the script (because it violates some RFC) or is it merely a workaround for a problem in other software (mail programs or Patchwork)? I know that this change is related to the issues that Patchwork has with identifying a patch series, but even with that background knowledge it is not clear why this fix is the right solution. > This change appends only one header pointing to very first patch > in series or to cover letter if available. > > [YOCTO #10718] > > Signed-off-by: Jose Lamego <jose.a.lam...@linux.intel.com> > --- > scripts/send-pull-request | 2 +- > 1 file changed, 1 insertion(+), 1 deletion(-) > > diff --git a/scripts/send-pull-request b/scripts/send-pull-request > index 575549d..a660c37 100755 > --- a/scripts/send-pull-request > +++ b/scripts/send-pull-request > @@ -162,7 +162,7 @@ PATCHES=$(echo $PDIR/*.patch) > if [ $AUTO_CL -eq 1 ]; then > # Send the cover letter to every recipient, both specified as well as > # harvested. Then remove it from the patches list. > - eval "git send-email $GIT_TO $GIT_CC $GIT_EXTRA_CC --confirm=always > --no-chain-reply-to --suppress-cc=all $CL" > + eval "git send-email $GIT_TO $GIT_CC $GIT_EXTRA_CC --confirm=always > --no-thread --suppress-cc=all $CL" > if [ $? -eq 1 ]; then > echo "ERROR: failed to send cover-letter with automatic > recipients." > exit 1 And I don't understand why this proposed change has the described effect. Does changing the threading parameters change the output of "git send-email" and thus indirectly the mail headers of the following patches? -- Best Regards, Patrick Ohly The content of this message is my personal opinion only and although I am an employee of Intel, the statements I make here in no way represent Intel's position on the issue, nor am I authorized to speak on behalf of Intel on this matter. -- ___ Openembedded-core mailing list Openembedded-core@lists.openembedded.org http://lists.openembedded.org/mailman/listinfo/openembedded-core
[OE-core] [PATCH 08/11] pybootchartgui: render disk space usage
This adds a new, separate chart showing the amount of disk space used
over time for each volume monitored during the build. The hight of the
graph entries represents the delta between current usage and minimal
usage during the build.
That's more useful than showing just the current usage, because then a
graph showing changes in the order of MBs in a volume that is several
GB large would be just flat.
The legend shows the maximum of those deltas, i.e. maximum amount of
space needed for the build. Minor caveat: sampling of disk space usage
starts a bit later than the initial task, so the displayed value may
be slightly lower than the actual amount of space needed because
sampling does not record the actual initial state.
Signed-off-by: Patrick Ohly <patrick.o...@intel.com>
---
scripts/pybootchartgui/pybootchartgui/draw.py| 62
scripts/pybootchartgui/pybootchartgui/parsing.py | 26 ++
scripts/pybootchartgui/pybootchartgui/samples.py | 11 +
3 files changed, 99 insertions(+)
diff --git a/scripts/pybootchartgui/pybootchartgui/draw.py
b/scripts/pybootchartgui/pybootchartgui/draw.py
index ec5dd33..f0143ad 100644
--- a/scripts/pybootchartgui/pybootchartgui/draw.py
+++ b/scripts/pybootchartgui/pybootchartgui/draw.py
@@ -133,6 +133,16 @@ TASK_COLOR_PACKAGE = (0.0, 1.00, 1.00, 1.0)
# Package Write RPM/DEB/IPK task color
TASK_COLOR_PACKAGE_WRITE = (0.0, 0.50, 0.50, 1.0)
+# Distinct colors used for different disk volumnes.
+# If we have more volumns, colors get re-used.
+VOLUME_COLORS = [
+ (1.0, 1.0, 0.00, 1.0),
+ (0.0, 1.00, 0.00, 1.0),
+ (1.0, 0.00, 1.00, 1.0),
+ (0.0, 0.00, 1.00, 1.0),
+ (0.0, 1.00, 1.00, 1.0),
+]
+
# Process states
STATE_UNDEFINED = 0
STATE_RUNNING = 1
@@ -397,6 +407,58 @@ def render_charts(ctx, options, clip, trace, curr_y, w, h,
sec_w):
curr_y = curr_y + 30 + bar_h
+ # render disk space usage
+ #
+ # Draws the amount of disk space used on each volume relative to the
+ # lowest recorded amount. The graphs for each volume are stacked above
+ # each other so that total disk usage is visible.
+ if trace.monitor_disk:
+ ctx.set_font_size(LEGEND_FONT_SIZE)
+ # Determine set of volumes for which we have
+ # information and the minimal amount of used disk
+ # space for each. Currently samples are allowed to
+ # not have a values for all volumes; drawing could be
+ # made more efficient if that wasn't the case.
+ volumes = set()
+ min_used = {}
+ for sample in trace.monitor_disk:
+ for volume, used in sample.records.items():
+ volumes.add(volume)
+ if volume not in min_used or min_used[volume] >
used:
+ min_used[volume] = used
+ volumes = sorted(list(volumes))
+ disk_scale = 0
+ for i, volume in enumerate(volumes):
+ volume_scale = max([sample.records[volume] -
min_used[volume]
+ for sample in trace.monitor_disk
+ if volume in sample.records])
+ # Does not take length of volume name into account, but
fixed offset
+ # works okay in practice.
+ draw_legend_box(ctx, '%s (max: %u MiB)' % (volume,
volume_scale / 1024 / 1024),
+ VOLUME_COLORS[i % len(VOLUME_COLORS)],
+ off_x + i * 250, curr_y+20, leg_s)
+ disk_scale += volume_scale
+
+ # render used amount of disk space
+ chart_rect = (off_x, curr_y+30, w, bar_h)
+ if clip_visible (clip, chart_rect):
+ draw_box_ticks (ctx, chart_rect, sec_w)
+ draw_annotations (ctx, proc_tree, trace.times,
chart_rect)
+ for i in range(len(volumes), 0, -1):
+ draw_chart (ctx, VOLUME_COLORS[(i - 1) %
len(VOLUME_COLORS)], True, chart_rect, \
+ [(sample.time,
+ # Sum up used space of all
volumes including the current one
+ # so that the graphs appear as
stacked on top of each other.
+ reduce(lambda x,y: x+y,
+[sample.records[volume] -
min_used[volume]
+ for volume in volumes[0:i]
+ if volume in
sample.records],
+0))
+f
[OE-core] [PATCH 07/11] pybootchartgui/parsing.py: fix error handling in meminfo parser
When matching fails, m.group(0) is invalid and can't be used in the
error message.
Signed-off-by: Patrick Ohly <patrick.o...@intel.com>
---
scripts/pybootchartgui/pybootchartgui/parsing.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/scripts/pybootchartgui/pybootchartgui/parsing.py
b/scripts/pybootchartgui/pybootchartgui/parsing.py
index af68435..48eb048 100644
--- a/scripts/pybootchartgui/pybootchartgui/parsing.py
+++ b/scripts/pybootchartgui/pybootchartgui/parsing.py
@@ -498,7 +498,7 @@ def _parse_proc_meminfo_log(file):
for line in lines:
match = meminfo_re.match(line)
if not match:
-raise ParseError("Invalid meminfo line \"%s\"" %
match.groups(0))
+raise ParseError("Invalid meminfo line \"%s\"" % line)
sample.add_value(match.group(1), int(match.group(2)))
if sample.valid():
--
2.1.4
--
___
Openembedded-core mailing list
Openembedded-core@lists.openembedded.org
http://lists.openembedded.org/mailman/listinfo/openembedded-core
[OE-core] [PATCH 10/11] buildstats: reduce amount of data stored for system utilization
Pre-processing /proc data during the build considerably reduces the
amount of data written to disk: 176KB instead of 4.7MB for a 20
minuted build. Parsing also becomes faster.
The disk monitor log added another 16KB in that example build. The
overall buildstat was 20MB, so the overhead for monitoring system
utilization is small enough that it can be enabled by default.
Signed-off-by: Patrick Ohly <patrick.o...@intel.com>
---
meta/lib/buildstats.py | 114 -
1 file changed, 104 insertions(+), 10 deletions(-)
diff --git a/meta/lib/buildstats.py b/meta/lib/buildstats.py
index 2499fb1..3cd333c 100644
--- a/meta/lib/buildstats.py
+++ b/meta/lib/buildstats.py
@@ -3,6 +3,7 @@
# like open log files and the time of the last sampling.
import time
+import re
import bb.event
class SystemStats:
@@ -12,14 +13,18 @@ class SystemStats:
bb.utils.mkdirhier(bsdir)
self.proc_files = []
-for filename in ('diskstats', 'meminfo', 'stat'):
+for filename, handler in (
+('diskstats', self._reduce_diskstats),
+('meminfo', self._reduce_meminfo),
+('stat', self._reduce_stat),
+):
# In practice, this class gets instantiated only once in
# the bitbake cooker process. Therefore 'append' mode is
# not strictly necessary, but using it makes the class
# more robust should two processes ever write
# concurrently.
-self.proc_files.append((filename,
-open(os.path.join(bsdir, 'proc_%s.log' %
filename), 'ab')))
+destfile = os.path.join(bsdir, '%sproc_%s.log' % ('reduced_' if
handler else '', filename))
+self.proc_files.append((filename, open(destfile, 'ab'), handler))
self.monitor_disk = open(os.path.join(bsdir, 'monitor_disk.log'), 'ab')
# Last time that we sampled /proc data resp. recorded disk monitoring
data.
self.last_proc = 0
@@ -30,18 +35,107 @@ class SystemStats:
# depends on the heartbeat event, which fires less often.
self.min_seconds = 1
+self.meminfo_regex =
re.compile(b'^(MemTotal|MemFree|Buffers|Cached|SwapTotal|SwapFree):\s*(\d+)')
+self.diskstats_regex =
re.compile(b'^([hsv]d.|mtdblock\d|mmcblk\d|cciss/c\d+d\d+.*)$')
+self.diskstats_ltime = None
+self.diskstats_data = None
+self.stat_ltimes = None
+
+def _reduce_meminfo(self, time, data):
+"""
+Extracts 'MemTotal', 'MemFree', 'Buffers', 'Cached', 'SwapTotal',
'SwapFree'
+and writes their values into a single line, in that order.
+"""
+values = {}
+for line in data.split(b'\n'):
+m = self.meminfo_regex.match(line)
+if m:
+values[m.group(1)] = m.group(2)
+if len(values) == 6:
+return (time,
+b' '.join([values[x] for x in
+ (b'MemTotal', b'MemFree', b'Buffers',
b'Cached', b'SwapTotal', b'SwapFree')]) + b'\n')
+
+def _diskstats_is_relevant_line(self, linetokens):
+if len(linetokens) != 14:
+return False
+disk = linetokens[2]
+return self.diskstats_regex.match(disk)
+
+def _reduce_diskstats(self, time, data):
+relevant_tokens = filter(self._diskstats_is_relevant_line, map(lambda
x: x.split(), data.split(b'\n')))
+diskdata = [0] * 3
+reduced = None
+for tokens in relevant_tokens:
+# rsect
+diskdata[0] += int(tokens[5])
+# wsect
+diskdata[1] += int(tokens[9])
+# use
+diskdata[2] += int(tokens[12])
+if self.diskstats_ltime:
+# We need to compute information about the time interval
+# since the last sampling and record the result as sample
+# for that point in the past.
+interval = time - self.diskstats_ltime
+if interval > 0:
+sums = [ a - b for a, b in zip(diskdata, self.diskstats_data) ]
+readTput = sums[0] / 2.0 * 100.0 / interval
+writeTput = sums[1] / 2.0 * 100.0 / interval
+util = float( sums[2] ) / 10 / interval
+util = max(0.0, min(1.0, util))
+reduced = (self.diskstats_ltime, (readTput, writeTput, util))
+
+self.diskstats_ltime = time
+self.diskstats_data = diskdata
+return reduced
+
+
+def _reduce_nop(self, time, data):
+return (time, data)
+
+def _reduce_stat(self, time, data):
+if not data:
+return None
+# CPU times {user, nice, system, idle, io_wait, irq, softirq} from
first line
+tokens = data.split(b'\n', 1)[0].split()
+times = [ int(token) for token in tokens[1:] ]
+reduced = None
+if self
[OE-core] [PATCH 09/11] pybootchartgui: simplify drawing of memory usage
The internal representation after parsing now matches exactly what the drawing code needs, thus speeding up drawing a bit. However, the main motivation is to store exactly that required information in a more compact file. Signed-off-by: Patrick Ohly <patrick.o...@intel.com> --- scripts/pybootchartgui/pybootchartgui/draw.py| 12 ++-- scripts/pybootchartgui/pybootchartgui/parsing.py | 2 +- scripts/pybootchartgui/pybootchartgui/samples.py | 16 3 files changed, 23 insertions(+), 7 deletions(-) diff --git a/scripts/pybootchartgui/pybootchartgui/draw.py b/scripts/pybootchartgui/pybootchartgui/draw.py index f0143ad..201ce45 100644 --- a/scripts/pybootchartgui/pybootchartgui/draw.py +++ b/scripts/pybootchartgui/pybootchartgui/draw.py @@ -463,25 +463,25 @@ def render_charts(ctx, options, clip, trace, curr_y, w, h, sec_w): chart_rect = (off_x, curr_y+30, w, meminfo_bar_h) mem_stats = trace.mem_stats if mem_stats and clip_visible (clip, chart_rect): - mem_scale = max(sample.records['MemTotal'] - sample.records['MemFree'] for sample in mem_stats) + mem_scale = max(sample.buffers for sample in mem_stats) draw_legend_box(ctx, "Mem cached (scale: %u MiB)" % (float(mem_scale) / 1024), MEM_CACHED_COLOR, off_x, curr_y+20, leg_s) draw_legend_box(ctx, "Used", MEM_USED_COLOR, off_x + 240, curr_y+20, leg_s) draw_legend_box(ctx, "Buffers", MEM_BUFFERS_COLOR, off_x + 360, curr_y+20, leg_s) - draw_legend_line(ctx, "Swap (scale: %u MiB)" % max([(sample.records['SwapTotal'] - sample.records['SwapFree'])/1024 for sample in mem_stats]), \ + draw_legend_line(ctx, "Swap (scale: %u MiB)" % max([(sample.swap)/1024 for sample in mem_stats]), \ MEM_SWAP_COLOR, off_x + 480, curr_y+20, leg_s) draw_box_ticks(ctx, chart_rect, sec_w) draw_annotations(ctx, proc_tree, trace.times, chart_rect) draw_chart(ctx, MEM_BUFFERS_COLOR, True, chart_rect, \ - [(sample.time, sample.records['MemTotal'] - sample.records['MemFree']) for sample in trace.mem_stats], \ + [(sample.time, sample.buffers) for sample in trace.mem_stats], \ proc_tree, [0, mem_scale]) draw_chart(ctx, MEM_USED_COLOR, True, chart_rect, \ - [(sample.time, sample.records['MemTotal'] - sample.records['MemFree'] - sample.records['Buffers']) for sample in mem_stats], \ + [(sample.time, sample.used) for sample in mem_stats], \ proc_tree, [0, mem_scale]) draw_chart(ctx, MEM_CACHED_COLOR, True, chart_rect, \ - [(sample.time, sample.records['Cached']) for sample in mem_stats], \ + [(sample.time, sample.cached) for sample in mem_stats], \ proc_tree, [0, mem_scale]) draw_chart(ctx, MEM_SWAP_COLOR, False, chart_rect, \ - [(sample.time, float(sample.records['SwapTotal'] - sample.records['SwapFree'])) for sample in mem_stats], \ + [(sample.time, float(sample.swap)) for sample in mem_stats], \ proc_tree, None) curr_y = curr_y + meminfo_bar_h diff --git a/scripts/pybootchartgui/pybootchartgui/parsing.py b/scripts/pybootchartgui/pybootchartgui/parsing.py index 301145a..1c8d8ef 100644 --- a/scripts/pybootchartgui/pybootchartgui/parsing.py +++ b/scripts/pybootchartgui/pybootchartgui/parsing.py @@ -503,7 +503,7 @@ def _parse_proc_meminfo_log(file): sample.add_value(match.group(1), int(match.group(2))) if sample.valid(): -mem_stats.append(sample) +mem_stats.append(DrawMemSample(sample)) return mem_stats diff --git a/scripts/pybootchartgui/pybootchartgui/samples.py b/scripts/pybootchartgui/pybootchartgui/samples.py index bedca41..9fc309b 100644 --- a/scripts/pybootchartgui/pybootchartgui/samples.py +++ b/scripts/pybootchartgui/pybootchartgui/samples.py @@ -53,6 +53,22 @@ class MemSample: # discard incomplete samples return [v for v in MemSample.used_values if v not in keys] == [] +class DrawMemSample: +""" +Condensed version of a MemSample with exactly the values used by the drawing code. +Initialized either from a valid MemSample or +a tuple/list of buffer/used/cached/swap values. +""" +def __init__(self, mem_sample): +self.time = mem_sample.time +if isinstance(mem_sample, MemSample): +self.buffers = mem_sample.records['MemTotal'] - mem_sample.records['MemFree'] +self.used = mem_sample.records['MemTotal'] - mem_sample.records['MemFree']
[OE-core] [PATCH 11/11] pybootchartgui: support reading reduced /proc logs
Pre-processing /proc data during the build considerably reduces the
amount of data written to disk: 176KB instead of 4.7MB for a 20
minuted build. Parsing also becomes faster.
buildstats.bbclass only writes the reduced logs now, but support for
the full /proc files is kept around as reference.
Signed-off-by: Patrick Ohly <patrick.o...@intel.com>
---
scripts/pybootchartgui/pybootchartgui/parsing.py | 31
1 file changed, 31 insertions(+)
diff --git a/scripts/pybootchartgui/pybootchartgui/parsing.py
b/scripts/pybootchartgui/pybootchartgui/parsing.py
index 1c8d8ef..bcfb2da 100644
--- a/scripts/pybootchartgui/pybootchartgui/parsing.py
+++ b/scripts/pybootchartgui/pybootchartgui/parsing.py
@@ -442,6 +442,12 @@ def _parse_proc_stat_log(file):
# skip the rest of statistics lines
return samples
+def _parse_reduced_log(file, sample_class):
+samples = []
+for time, lines in _parse_timed_blocks(file):
+samples.append(sample_class(time, *[float(x) for x in
lines[0].split()]))
+return samples
+
def _parse_proc_disk_stat_log(file):
"""
Parse file for disk stats, but only look at the whole device, eg. sda,
@@ -483,6 +489,25 @@ def _parse_proc_disk_stat_log(file):
return disk_stats
+def _parse_reduced_proc_meminfo_log(file):
+"""
+Parse file for global memory statistics with
+'MemTotal', 'MemFree', 'Buffers', 'Cached', 'SwapTotal', 'SwapFree' values
+(in that order) directly stored on one line.
+"""
+used_values = ('MemTotal', 'MemFree', 'Buffers', 'Cached', 'SwapTotal',
'SwapFree',)
+
+mem_stats = []
+for time, lines in _parse_timed_blocks(file):
+sample = MemSample(time)
+for name, value in zip(used_values, lines[0].split()):
+sample.add_value(name, int(value))
+
+if sample.valid():
+mem_stats.append(DrawMemSample(sample))
+
+return mem_stats
+
def _parse_proc_meminfo_log(file):
"""
Parse file for global memory statistics.
@@ -702,10 +727,16 @@ def _do_parse(writer, state, filename, file):
name = os.path.basename(filename)
if name == "proc_diskstats.log":
state.disk_stats = _parse_proc_disk_stat_log(file)
+elif name == "reduced_proc_diskstats.log":
+state.disk_stats = _parse_reduced_log(file, DiskSample)
elif name == "proc_stat.log":
state.cpu_stats = _parse_proc_stat_log(file)
+elif name == "reduced_proc_stat.log":
+state.cpu_stats = _parse_reduced_log(file, CPUSample)
elif name == "proc_meminfo.log":
state.mem_stats = _parse_proc_meminfo_log(file)
+elif name == "reduced_proc_meminfo.log":
+state.mem_stats = _parse_reduced_proc_meminfo_log(file)
elif name == "cmdline2.log":
state.cmdline = _parse_cmdline_log(writer, file)
elif name == "monitor_disk.log":
--
2.1.4
--
___
Openembedded-core mailing list
Openembedded-core@lists.openembedded.org
http://lists.openembedded.org/mailman/listinfo/openembedded-core
[OE-core] [PATCH 06/11] buildstats: record disk space usage
Hooks into the new monitordisk.py event and records the used space for
each volume. That is probably the only relevant value when it comes to
visualizing the build and recording more would only increase disk
usage.
Signed-off-by: Patrick Ohly <patrick.o...@intel.com>
---
meta/classes/buildstats.bbclass | 4 ++--
meta/lib/buildstats.py | 22 +-
2 files changed, 19 insertions(+), 7 deletions(-)
diff --git a/meta/classes/buildstats.bbclass b/meta/classes/buildstats.bbclass
index 2abc1a7..113a246 100644
--- a/meta/classes/buildstats.bbclass
+++ b/meta/classes/buildstats.bbclass
@@ -199,8 +199,8 @@ python runqueue_stats () {
system_stats = buildstats.get_system_stats(d, init=init)
if system_stats:
# Ensure that we sample at important events.
-system_stats.sample(force=isinstance(e, bb.event.BuildCompleted))
+system_stats.sample(e, force=isinstance(e, bb.event.BuildCompleted))
}
addhandler runqueue_stats
-runqueue_stats[eventmask] = "bb.runqueue.runQueueTaskStarted
bb.event.HeartbeatEvent bb.event.BuildCompleted"
+runqueue_stats[eventmask] = "bb.runqueue.runQueueTaskStarted
bb.event.HeartbeatEvent bb.event.BuildCompleted bb.event.MonitorDiskEvent"
diff --git a/meta/lib/buildstats.py b/meta/lib/buildstats.py
index 1664c52..2499fb1 100644
--- a/meta/lib/buildstats.py
+++ b/meta/lib/buildstats.py
@@ -3,6 +3,7 @@
# like open log files and the time of the last sampling.
import time
+import bb.event
class SystemStats:
def __init__(self, d):
@@ -19,17 +20,19 @@ class SystemStats:
# concurrently.
self.proc_files.append((filename,
open(os.path.join(bsdir, 'proc_%s.log' %
filename), 'ab')))
-# Last time that we sampled data.
-self.last = 0
+self.monitor_disk = open(os.path.join(bsdir, 'monitor_disk.log'), 'ab')
+# Last time that we sampled /proc data resp. recorded disk monitoring
data.
+self.last_proc = 0
+self.last_disk_monitor = 0
# Minimum number of seconds between recording a sample. This
# becames relevant when we get called very often while many
# short tasks get started. Sampling during quiet periods
# depends on the heartbeat event, which fires less often.
self.min_seconds = 1
-def sample(self, force):
+def sample(self, event, force):
now = time.time()
-if (now - self.last > self.min_seconds) or force:
+if (now - self.last_proc > self.min_seconds) or force:
for filename, output in self.proc_files:
with open(os.path.join('/proc', filename), 'rb') as input:
data = input.read()
@@ -39,7 +42,16 @@ class SystemStats:
('%.0f\n' % now).encode('ascii') +
data +
b'\n')
-self.last = now
+self.last_proc = now
+
+if isinstance(event, bb.event.MonitorDiskEvent) and \
+ ((now - self.last_disk_monitor > self.min_seconds) or force):
+os.write(self.monitor_disk.fileno(),
+ ('%.0f\n' % now).encode('ascii') +
+ ''.join(['%s: %d\n' % (dev, sample.total_bytes -
sample.free_bytes)
+ for dev, sample in
event.disk_usage.items()]).encode('ascii') +
+ b'\n')
+self.last_disk_monitor = now
_system_stats = None
--
2.1.4
--
___
Openembedded-core mailing list
Openembedded-core@lists.openembedded.org
http://lists.openembedded.org/mailman/listinfo/openembedded-core
[OE-core] [PATCH 04/11] pybootchartgui: show system utilization
This enables rendering of the original bootchart charts for CPU, disk
and memory usage. It depends on the /proc samples recorded by the
updated buildstats.bbclass. Currently, empty charts CPU and disk usage
charts are drawn if that data is not present; the memory chart already
gets skipped when there's no data, which will also have to be added
for the other two.
Signed-off-by: Patrick Ohly <patrick.o...@intel.com>
---
scripts/pybootchartgui/pybootchartgui/draw.py| 16 +--
scripts/pybootchartgui/pybootchartgui/parsing.py | 61 +---
2 files changed, 57 insertions(+), 20 deletions(-)
diff --git a/scripts/pybootchartgui/pybootchartgui/draw.py
b/scripts/pybootchartgui/pybootchartgui/draw.py
index 925002d..bddd804 100644
--- a/scripts/pybootchartgui/pybootchartgui/draw.py
+++ b/scripts/pybootchartgui/pybootchartgui/draw.py
@@ -321,6 +321,16 @@ def extents(options, xscale, trace):
w = int ((end - start) * sec_w_base * xscale) + 2 * off_x
h = proc_h * processes + header_h + 2 * off_y
+ if options.charts:
+ if trace.cpu_stats:
+ h += 30 + bar_h
+ if trace.disk_stats:
+ h += 30 + bar_h
+ if trace.monitor_disk:
+ h += 30 + bar_h
+ if trace.mem_stats:
+ h += meminfo_bar_h
+
return (w, h)
def clip_visible(clip, rect):
@@ -496,6 +506,9 @@ def render(ctx, options, xscale, trace):
w -= 2*off_x
curr_y = off_y;
+ if options.charts:
+ curr_y = render_charts (ctx, options, clip, trace, curr_y, w,
h, sec_w)
+
curr_y = render_processes_chart (ctx, options, trace, curr_y, w, h,
sec_w)
return
@@ -513,9 +526,6 @@ def render(ctx, options, xscale, trace):
else:
curr_y = off_y;
- if options.charts:
- curr_y = render_charts (ctx, options, clip, trace, curr_y, w,
h, sec_w)
-
# draw process boxes
proc_height = h
if proc_tree.taskstats and options.cumulative:
diff --git a/scripts/pybootchartgui/pybootchartgui/parsing.py
b/scripts/pybootchartgui/pybootchartgui/parsing.py
index a3a0b0b..af68435 100644
--- a/scripts/pybootchartgui/pybootchartgui/parsing.py
+++ b/scripts/pybootchartgui/pybootchartgui/parsing.py
@@ -38,16 +38,17 @@ class Trace:
self.min = None
self.max = None
self.headers = None
-self.disk_stats = None
+self.disk_stats = []
self.ps_stats = None
self.taskstats = None
-self.cpu_stats = None
+self.cpu_stats = []
self.cmdline = None
self.kernel = None
self.kernel_tree = None
self.filename = None
self.parent_map = None
-self.mem_stats = None
+self.mem_stats = []
+self.times = [] # Always empty, but expected by draw.py when drawing
system charts.
if len(paths):
parse_paths (writer, self, paths)
@@ -58,6 +59,19 @@ class Trace:
self.min = min(self.start.keys())
self.max = max(self.end.keys())
+
+# Rendering system charts depends on start and end
+# time. Provide them where the original drawing code expects
+# them, i.e. in proc_tree.
+class BitbakeProcessTree:
+def __init__(self, start_time, end_time):
+self.start_time = start_time
+self.end_time = end_time
+self.duration = self.end_time - self.start_time
+self.proc_tree = BitbakeProcessTree(min(self.start.keys()),
+max(self.end.keys()))
+
+
return
# Turn that parsed information into something more useful
@@ -427,7 +441,7 @@ def _parse_proc_stat_log(file):
# skip the rest of statistics lines
return samples
-def _parse_proc_disk_stat_log(file, numCpu):
+def _parse_proc_disk_stat_log(file):
"""
Parse file for disk stats, but only look at the whole device, eg. sda,
not sda1, sda2 etc. The format of relevant lines should be:
@@ -462,7 +476,7 @@ def _parse_proc_disk_stat_log(file, numCpu):
sums = [ a - b for a, b in zip(sample1.diskdata, sample2.diskdata) ]
readTput = sums[0] / 2.0 * 100.0 / interval
writeTput = sums[1] / 2.0 * 100.0 / interval
-util = float( sums[2] ) / 10 / interval / numCpu
+util = float( sums[2] ) / 10 / interval
util = max(0.0, min(1.0, util))
disk_stats.append(DiskSample(sample2.time, readTput, writeTput, util))
@@ -628,6 +642,20 @@ def _parse_cmdline_log(writer, file):
cmdLines[pid] = values
return cmdLines
+def _parse_bitbake_buildstats(writer, state, filename, file):
+paths = filename.split("/")
+task = paths[-1]
+pn = paths[-2]
+start = None
+end = None
+for line in file:
+
[OE-core] [PATCH 05/11] pybootchartgui/draw.py: skip empty CPU and disk usage charts
The only real change is the addition of two if checks that skips the corresponding drawing code when there is no data. Signed-off-by: Patrick Ohly <patrick.o...@intel.com> --- scripts/pybootchartgui/pybootchartgui/draw.py | 98 ++- 1 file changed, 50 insertions(+), 48 deletions(-) diff --git a/scripts/pybootchartgui/pybootchartgui/draw.py b/scripts/pybootchartgui/pybootchartgui/draw.py index bddd804..ec5dd33 100644 --- a/scripts/pybootchartgui/pybootchartgui/draw.py +++ b/scripts/pybootchartgui/pybootchartgui/draw.py @@ -344,56 +344,58 @@ def render_charts(ctx, options, clip, trace, curr_y, w, h, sec_w): proc_tree = options.proc_tree(trace) # render bar legend - ctx.set_font_size(LEGEND_FONT_SIZE) - - draw_legend_box(ctx, "CPU (user+sys)", CPU_COLOR, off_x, curr_y+20, leg_s) - draw_legend_box(ctx, "I/O (wait)", IO_COLOR, off_x + 120, curr_y+20, leg_s) - - # render I/O wait - chart_rect = (off_x, curr_y+30, w, bar_h) - if clip_visible (clip, chart_rect): - draw_box_ticks (ctx, chart_rect, sec_w) - draw_annotations (ctx, proc_tree, trace.times, chart_rect) - draw_chart (ctx, IO_COLOR, True, chart_rect, \ - [(sample.time, sample.user + sample.sys + sample.io) for sample in trace.cpu_stats], \ - proc_tree, None) - # render CPU load - draw_chart (ctx, CPU_COLOR, True, chart_rect, \ - [(sample.time, sample.user + sample.sys) for sample in trace.cpu_stats], \ - proc_tree, None) - - curr_y = curr_y + 30 + bar_h + if trace.cpu_stats: + ctx.set_font_size(LEGEND_FONT_SIZE) + + draw_legend_box(ctx, "CPU (user+sys)", CPU_COLOR, off_x, curr_y+20, leg_s) + draw_legend_box(ctx, "I/O (wait)", IO_COLOR, off_x + 120, curr_y+20, leg_s) + + # render I/O wait + chart_rect = (off_x, curr_y+30, w, bar_h) + if clip_visible (clip, chart_rect): + draw_box_ticks (ctx, chart_rect, sec_w) + draw_annotations (ctx, proc_tree, trace.times, chart_rect) + draw_chart (ctx, IO_COLOR, True, chart_rect, \ + [(sample.time, sample.user + sample.sys + sample.io) for sample in trace.cpu_stats], \ + proc_tree, None) + # render CPU load + draw_chart (ctx, CPU_COLOR, True, chart_rect, \ + [(sample.time, sample.user + sample.sys) for sample in trace.cpu_stats], \ + proc_tree, None) + + curr_y = curr_y + 30 + bar_h # render second chart - draw_legend_line(ctx, "Disk throughput", DISK_TPUT_COLOR, off_x, curr_y+20, leg_s) - draw_legend_box(ctx, "Disk utilization", IO_COLOR, off_x + 120, curr_y+20, leg_s) - -# render I/O utilization - chart_rect = (off_x, curr_y+30, w, bar_h) - if clip_visible (clip, chart_rect): - draw_box_ticks (ctx, chart_rect, sec_w) - draw_annotations (ctx, proc_tree, trace.times, chart_rect) - draw_chart (ctx, IO_COLOR, True, chart_rect, \ - [(sample.time, sample.util) for sample in trace.disk_stats], \ - proc_tree, None) - - # render disk throughput - max_sample = max (trace.disk_stats, key = lambda s: s.tput) - if clip_visible (clip, chart_rect): - draw_chart (ctx, DISK_TPUT_COLOR, False, chart_rect, \ - [(sample.time, sample.tput) for sample in trace.disk_stats], \ - proc_tree, None) - - pos_x = off_x + ((max_sample.time - proc_tree.start_time) * w / proc_tree.duration) - - shift_x, shift_y = -20, 20 - if (pos_x < off_x + 245): - shift_x, shift_y = 5, 40 - - label = "%dMB/s" % round ((max_sample.tput) / 1024.0) - draw_text (ctx, label, DISK_TPUT_COLOR, pos_x + shift_x, curr_y + shift_y) - - curr_y = curr_y + 30 + bar_h + if trace.disk_stats: + draw_legend_line(ctx, "Disk throughput", DISK_TPUT_COLOR, off_x, curr_y+20, leg_s) + draw_legend_box(ctx, "Disk utilization", IO_COLOR, off_x + 120, curr_y+20, leg_s) + + # render I/O utilization + chart_rect = (off_x, curr_y+30, w, bar_h) + if clip_visible (clip, chart_rect): + draw_box_ticks (ctx, chart_rect, sec_w) + draw_annotations (ctx, proc_tree, trace.times, chart_rect) + draw_chart (ctx, IO_COLOR, True, chart_rect
[OE-core] [PATCH 00/11] system statistics sampling
These patches depend on the corresponding patches to bitbake.
buildstats.bbclass gets extended so that disk bandwidth and capacity,
CPU and memory usage get sampled at regular time intervals by
default. pybootchart had code for most of that data, which gets
restored and re-enabled. The disk capacity graph is new.
The original pybootchart just stored raw dumps of the special files
under /proc. bitbake builds run longer, so a more compact ("reduced")
format is used.
Logging disk capacity was mostly motivated by some work on enhancing
rm_work.bbclass behavior, but could also be useful for others ("how
much free space did my build need").
The following changes since commit 9f1fe76727e98e58fc9e46ea2b49cf5c0cb48e6c:
libpcap: Fix build when PACKAGECONFIG ipv6 is not enable (2016-11-23 11:02:33
+)
are available in the git repository at:
git://github.com/pohly/openembedded-core buildstats
https://github.com/pohly/openembedded-core/tree/buildstats
Patrick Ohly (11):
buildstats: add system state sampling
pybootchartgui/draw.py: allow moving process chart up and down
pybootchartgui/draw.py: fix drawing of samples not starting at zero
pybootchartgui: show system utilization
pybootchartgui/draw.py: skip empty CPU and disk usage charts
buildstats: record disk space usage
pybootchartgui/parsing.py: fix error handling in meminfo parser
pybootchartgui: render disk space usage
pybootchartgui: simplify drawing of memory usage
buildstats: reduce amount of data stored for system utilization
pybootchartgui: support reading reduced /proc logs
meta/classes/buildstats.bbclass | 16 ++
meta/lib/buildstats.py | 156 ++
scripts/pybootchartgui/pybootchartgui/draw.py| 192 ---
scripts/pybootchartgui/pybootchartgui/parsing.py | 122 +++---
scripts/pybootchartgui/pybootchartgui/samples.py | 27
5 files changed, 435 insertions(+), 78 deletions(-)
create mode 100644 meta/lib/buildstats.py
--
2.1.4
--
___
Openembedded-core mailing list
Openembedded-core@lists.openembedded.org
http://lists.openembedded.org/mailman/listinfo/openembedded-core
[OE-core] [PATCH 03/11] pybootchartgui/draw.py: fix drawing of samples not starting at zero
The code did not handle x scaling correctly when drawing starts at some time larger than zero, i.e. it worked for normal bootchart data, but not for the system statistics recorded by buildstats.bbclass. Signed-off-by: Patrick Ohly <patrick.o...@intel.com> --- scripts/pybootchartgui/pybootchartgui/draw.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/pybootchartgui/pybootchartgui/draw.py b/scripts/pybootchartgui/pybootchartgui/draw.py index 2b5907b..925002d 100644 --- a/scripts/pybootchartgui/pybootchartgui/draw.py +++ b/scripts/pybootchartgui/pybootchartgui/draw.py @@ -256,7 +256,7 @@ def draw_chart(ctx, color, fill, chart_bounds, data, proc_tree, data_range): # avoid divide by zero if max_y == 0: max_y = 1.0 - xscale = float (chart_bounds[2]) / max_x + xscale = float (chart_bounds[2]) / (max_x - x_shift) # If data_range is given, scale the chart so that the value range in # data_range matches the chart bounds exactly. # Otherwise, scale so that the actual data matches the chart bounds. -- 2.1.4 -- ___ Openembedded-core mailing list Openembedded-core@lists.openembedded.org http://lists.openembedded.org/mailman/listinfo/openembedded-core
[OE-core] [PATCH 01/11] buildstats: add system state sampling
/proc/[diskstats|meminfo|stat] get sampled and written to the same
proc_.log files as during normal bootchat logging. This will
allow rendering the CPU, disk and memory usage charts.
Right now sampling happens once a second, triggered by the heartbeat
event.That produces quite a bit of data for long builds, which will be
addressed in a separate commit by storing the data in a more compact
form.
Signed-off-by: Patrick Ohly <patrick.o...@intel.com>
---
meta/classes/buildstats.bbclass | 16 +
meta/lib/buildstats.py | 50 +
2 files changed, 66 insertions(+)
create mode 100644 meta/lib/buildstats.py
diff --git a/meta/classes/buildstats.bbclass b/meta/classes/buildstats.bbclass
index 57ecc8f..2abc1a7 100644
--- a/meta/classes/buildstats.bbclass
+++ b/meta/classes/buildstats.bbclass
@@ -188,3 +188,19 @@ python run_buildstats () {
addhandler run_buildstats
run_buildstats[eventmask] = "bb.event.BuildStarted bb.event.BuildCompleted
bb.build.TaskStarted bb.build.TaskSucceeded bb.build.TaskFailed"
+python runqueue_stats () {
+import buildstats
+from bb import event, runqueue
+# We should not record any samples before the first task has started,
+# because that's the first activity shown in the process chart.
+# Besides, at that point we are sure that the build variables
+# are available that we need to find the output directory.
+init = isinstance(e, bb.runqueue.runQueueTaskStarted)
+system_stats = buildstats.get_system_stats(d, init=init)
+if system_stats:
+# Ensure that we sample at important events.
+system_stats.sample(force=isinstance(e, bb.event.BuildCompleted))
+}
+
+addhandler runqueue_stats
+runqueue_stats[eventmask] = "bb.runqueue.runQueueTaskStarted
bb.event.HeartbeatEvent bb.event.BuildCompleted"
diff --git a/meta/lib/buildstats.py b/meta/lib/buildstats.py
new file mode 100644
index 000..1664c52
--- /dev/null
+++ b/meta/lib/buildstats.py
@@ -0,0 +1,50 @@
+# Implements system state sampling. Called by buildstats.bbclass.
+# Because it is a real Python module, it can hold persistent state,
+# like open log files and the time of the last sampling.
+
+import time
+
+class SystemStats:
+def __init__(self, d):
+bn = d.getVar('BUILDNAME', True)
+bsdir = os.path.join(d.getVar('BUILDSTATS_BASE', True), bn)
+bb.utils.mkdirhier(bsdir)
+
+self.proc_files = []
+for filename in ('diskstats', 'meminfo', 'stat'):
+# In practice, this class gets instantiated only once in
+# the bitbake cooker process. Therefore 'append' mode is
+# not strictly necessary, but using it makes the class
+# more robust should two processes ever write
+# concurrently.
+self.proc_files.append((filename,
+open(os.path.join(bsdir, 'proc_%s.log' %
filename), 'ab')))
+# Last time that we sampled data.
+self.last = 0
+# Minimum number of seconds between recording a sample. This
+# becames relevant when we get called very often while many
+# short tasks get started. Sampling during quiet periods
+# depends on the heartbeat event, which fires less often.
+self.min_seconds = 1
+
+def sample(self, force):
+now = time.time()
+if (now - self.last > self.min_seconds) or force:
+for filename, output in self.proc_files:
+with open(os.path.join('/proc', filename), 'rb') as input:
+data = input.read()
+# Unbuffered raw write, less overhead and useful
+# in case that we end up with concurrent writes.
+os.write(output.fileno(),
+ ('%.0f\n' % now).encode('ascii') +
+ data +
+ b'\n')
+self.last = now
+
+_system_stats = None
+
+def get_system_stats(d, init):
+global _system_stats
+if not _system_stats and init:
+_system_stats = SystemStats(d)
+return _system_stats
--
2.1.4
--
___
Openembedded-core mailing list
Openembedded-core@lists.openembedded.org
http://lists.openembedded.org/mailman/listinfo/openembedded-core
[OE-core] [PATCH 02/11] pybootchartgui/draw.py: allow moving process chart up and down
Substracting curr_y when determining the hight of the process chart is wrong because the height is independent of the position where the chart is about to be drawn. It happens to work at the moment because curr_y is always 10 when render_processes_chart() gets called. But it leads to a negative height when other charts are drawn above it, and then the grid gets drawn on top of those other charts. Substracting some constant is relevant because otherwise the box is slightly larger than the process bars. Not sure exactly where that comes from (text height?); leg_s seems a suitable constant and happens to be 10, so everything still gets rendered exactly as before. Signed-off-by: Patrick Ohly <patrick.o...@intel.com> --- scripts/pybootchartgui/pybootchartgui/draw.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/pybootchartgui/pybootchartgui/draw.py b/scripts/pybootchartgui/pybootchartgui/draw.py index 8c574be..2b5907b 100644 --- a/scripts/pybootchartgui/pybootchartgui/draw.py +++ b/scripts/pybootchartgui/pybootchartgui/draw.py @@ -415,7 +415,7 @@ def render_charts(ctx, options, clip, trace, curr_y, w, h, sec_w): return curr_y def render_processes_chart(ctx, options, trace, curr_y, w, h, sec_w): -chart_rect = [off_x, curr_y+header_h, w, h - 2 * off_y - (curr_y+header_h) + proc_h] +chart_rect = [off_x, curr_y+header_h, w, h - 2 * off_y - header_h - leg_s + proc_h] draw_legend_box (ctx, "Configure", \ TASK_COLOR_CONFIGURE, off_x , curr_y + 45, leg_s) -- 2.1.4 -- ___ Openembedded-core mailing list Openembedded-core@lists.openembedded.org http://lists.openembedded.org/mailman/listinfo/openembedded-core
Re: [OE-core] ASSUME_PROVIDED versus SANITY_REQUIRED_UTILITIES versus "The Build Host Packages"
On Mon, 2016-11-28 at 12:04 +, Richard Purdie wrote: > On Mon, 2016-11-28 at 06:20 -0500, Robert P. J. Day wrote: > > a bit confused by what i'm seeing in a recent > > qemuppc/core-image-minimal build on my fedora system regarding which > > native packages are built, despite what's in bitbake.conf (using > > current poky layer checkout). > > > > first, here's a snippet from bitbake.conf: > > > > ASSUME_PROVIDED = "\ > > bzip2-native \ <--- > > chrpath-native \ > > file-native \ <--- > > findutils-native \ > > git-native \ > > grep-native \ > > diffstat-native \ > > ... snip ... > > > > suggesting that (among other things) bzip2-native and file-native > > shouldn't be built -- it's the developer's responsibility to install > > them, yes? but if i peek under tmp/work/x86_64-linux, i can see: > > There are two ways "file-native" can be used in a build. It can be used > as the host provided "file" command and it is also needed by the file > recipe to build file for the target. I recently ran into a third usage of "file-native": swupd-server links against libmagic from file and therefore has a DEPENDS = "file". But building swupd-server-native didn't actually build file because of ASSUME_PROVIDED and because I hadn't installed libmagic-dev on my build host, the build was failing. Is there a way to declare that ASSUME_PROVIDED does not apply to this case? It sounds like there is a way (based on your comments about building file-native when building file and the libbz2-devel example), but it did not become clear to me how that works in practice. Or is it just a case of educating the developer that libmagic-dev needs to be installed on the build host in addition to the file command? -- Best Regards, Patrick Ohly The content of this message is my personal opinion only and although I am an employee of Intel, the statements I make here in no way represent Intel's position on the issue, nor am I authorized to speak on behalf of Intel on this matter. -- ___ Openembedded-core mailing list Openembedded-core@lists.openembedded.org http://lists.openembedded.org/mailman/listinfo/openembedded-core
Re: [OE-core] [PATCH v1] wic: Add --exclude-path option to rootfs source plugin.
Hi!
Wow, that was fast :-)
On Fri, 2016-11-25 at 11:15 +0100, Kristian Amlie wrote:
> +if os.stat(real_rootfs_dir).st_dev ==
> os.stat(cr_workdir).st_dev:
> +# Optimization if both directories are on the same
> file system:
> +# copy using hardlinks.
> +cp_args = "-al"
> +else:
> +cp_args = "-a"
> +exec_cmd("cp %s %s %s" % (cp_args, real_rootfs_dir,
> new_rootfs))
Not a full review (I'll leave that to Ed), just one thing which caught
my eye: when the rootfs contains xattrs, they get lost here.
Use oe.path.copyhardlinktree() instead, it also does the hardlinking
trick.
--
Best Regards, Patrick Ohly
The content of this message is my personal opinion only and although
I am an employee of Intel, the statements I make here in no way
represent Intel's position on the issue, nor am I authorized to speak
on behalf of Intel on this matter.
--
___
Openembedded-core mailing list
Openembedded-core@lists.openembedded.org
http://lists.openembedded.org/mailman/listinfo/openembedded-core
Re: [OE-core] Contents of non-rootfs partitions
On Thu, 2016-11-24 at 16:51 +0100, Kristian Amlie wrote: > On 24/11/16 16:28, Andreas Oberritter wrote: > > Note that in a shell, "data/*" doesn't include "data/.*". So this syntax > > avoids the confusing rsync syntax in trade for another one. I'd prefer > > the rsync flavour, because "behaves like rsync" is easier to remember > > than "behaves like a shell glob, but differently". Confusion could be > > prevented with good documentation, FWIW. > > This is turning into a bit of a bikeshed discussion. I have a slight > preference for the glob version, because I think it's less surprising, > but I'm fine either way. > > I count myself and Ed in favor of glob version, Ulrich and Andreas in > favor of rsync version. Patrick, you wanna be tie breaker? :-) But I said I was undecided ;-} Okay, I vote for the rsync flavor. -- Best Regards, Patrick Ohly The content of this message is my personal opinion only and although I am an employee of Intel, the statements I make here in no way represent Intel's position on the issue, nor am I authorized to speak on behalf of Intel on this matter. -- ___ Openembedded-core mailing list Openembedded-core@lists.openembedded.org http://lists.openembedded.org/mailman/listinfo/openembedded-core
Re: [OE-core] Contents of non-rootfs partitions
On Thu, 2016-11-24 at 16:51 +0200, Ed Bartosh wrote: > On Thu, Nov 24, 2016 at 03:43:18PM +0100, Kristian Amlie wrote: > > On 24/11/16 14:23, Ed Bartosh wrote: > > > Would this way be less intuitive? > > > --exclude-path data/* > > > --exclude-path data > > > > > > We can go even further with it allowing any level of directories: > > > --exclude-path data/tmp/* Just to clarify, that is meant to also match data/tmp/.hidden-file, right? I.e. fnmatch() without the special FNM_PERIOD flag. > > > --exclude-path data/db/tmp > > > ... > > > > I agree, this is pretty unambiguous and easy to understand. > > > > But this raises the question: Should we go all the way and support > > wildcards? Which might make it a bit complicated. Maybe support only > > pure '*' for now? > > As it shouldn't be hard to implement I'd go for it. Additional code implies additional testing. Remember that it also should better be supported by mkfs.ext4 and friends. I'd rather start simple and only add additional complexity when needed. -- Best Regards, Patrick Ohly The content of this message is my personal opinion only and although I am an employee of Intel, the statements I make here in no way represent Intel's position on the issue, nor am I authorized to speak on behalf of Intel on this matter. -- ___ Openembedded-core mailing list Openembedded-core@lists.openembedded.org http://lists.openembedded.org/mailman/listinfo/openembedded-core
Re: [OE-core] [PATCH 2/2] base-passwd: set root's default password to 'root'
On Thu, 2016-11-24 at 09:09 -0500, Philip Balister wrote: > On 11/24/2016 02:46 AM, Patrick Ohly wrote: > > On Thu, 2016-11-24 at 11:38 +0800, Robert Yang wrote: > >> Currently, debug-tweaks is in EXTRA_IMAGE_FEATURES by default for poky, and > >> there is no passwd, so that user can login easily without a passwd, I think > >> that current status is more unsafe ? > > > > Both well-known password and no password are unsafe. User "root" with > > password "root" is not even "more" safe already now, because tools that > > brute-force logins try that. Choosing something else would be a bit > > safer for a short while until the tools add it to their dictionary. > > > > Poky is also targeting a different audience than OE-core. Poky can > > assume to be used in a secure environment, OE-core can't (because it > > might be used for all kinds of devices). > > > > That is the first time I've heard Poky is targeting an audience assumed > to be running in a secure environment. At least the default local.conf seems to be meant for that (easy-of-use for developers preferred over security in a hostile environment). > Should we document what Poky this > somewhere? From where I sit, this seems to be an odd limitation. I'm not aware of a document explicitly documenting this either. I wouldn't call it a limitation, though: a real product could be built with a configuration that doesn't enable debug-tweaks. As Paul said before, more documentation about first boot, login mechanisms, security considerations, etc. certainly would be useful. -- Best Regards, Patrick Ohly The content of this message is my personal opinion only and although I am an employee of Intel, the statements I make here in no way represent Intel's position on the issue, nor am I authorized to speak on behalf of Intel on this matter. -- ___ Openembedded-core mailing list Openembedded-core@lists.openembedded.org http://lists.openembedded.org/mailman/listinfo/openembedded-core
Re: [OE-core] [PATCH] meta/conf/layer.conf: Add recommended download layer
On Wed, 2016-11-23 at 13:42 -0600, Mark Hatle wrote: > On 11/23/16 12:56 PM, Martin Jansa wrote: > > "Have all patches applied to BitBake and OpenEmbedded-Core (if present) > > been submitted to the open source community?" > > > > Shouldn't the wording be change to something like "all applicable > > patches" or "all generally useful patches"? > > > > It seems strange to send project specific patches together with cover > > saying that they aren't generally applicable and shouldn't be merged, > > just because of this requirement. > > It was done this way to prevent people from cheating and claiming something > was > (effectively) not useful/applicable/etc when it some special secret sauce. Perhaps one could use [FYI] instead of [PATCH] in the mail subject to make the intention even clearer, and patchwork could get configured to not track such patches or at least not show them as new? Just a thought. As it stands now, there's an entry for this patch that someone (Richard?) will have to close: https://patchwork.openembedded.org/series/4056/ -- Best Regards, Patrick Ohly The content of this message is my personal opinion only and although I am an employee of Intel, the statements I make here in no way represent Intel's position on the issue, nor am I authorized to speak on behalf of Intel on this matter. -- ___ Openembedded-core mailing list Openembedded-core@lists.openembedded.org http://lists.openembedded.org/mailman/listinfo/openembedded-core
[OE-core] [PATCH v2] pseudo: include fix for xattr corruption
pseudo_1.8.1.bb gets the backported patch and pseudo_git.bb gets
updated to include the commit.
Signed-off-by: Patrick Ohly <patrick.o...@intel.com>
---
Notes:
Change since V1:
Updated the commit message. It was still the one written
for the initial, broken upstream patch for the problem.
The attached patch works.
.../pseudo/files/More-correctly-fix-xattrs.patch | 37 ++
meta/recipes-devtools/pseudo/pseudo_1.8.1.bb | 1 +
meta/recipes-devtools/pseudo/pseudo_git.bb | 2 +-
3 files changed, 39 insertions(+), 1 deletion(-)
create mode 100644
meta/recipes-devtools/pseudo/files/More-correctly-fix-xattrs.patch
diff --git a/meta/recipes-devtools/pseudo/files/More-correctly-fix-xattrs.patch
b/meta/recipes-devtools/pseudo/files/More-correctly-fix-xattrs.patch
new file mode 100644
index 000..3d178f9
--- /dev/null
+++ b/meta/recipes-devtools/pseudo/files/More-correctly-fix-xattrs.patch
@@ -0,0 +1,37 @@
+From 45eca34c754d416a38bee90fb2d3c110a0b6cc5f Mon Sep 17 00:00:00 2001
+From: Seebs <se...@seebs.net>
+Date: Thu, 3 Nov 2016 11:36:12 -0500
+Subject: [PATCH] More-correctly fix xattrs
+
+Fix provided by Patrick Ohly <patrick.o...@intel.com>. This resolves
+the actual cause of the path length mismatches, and explains why
+I couldn't quite explain why the previous one had only sometimes
+worked, also why it showed up on directories but not plain files.
+
+Signed-off-by: Seebs <se...@seebs.net>
+
+Fixes [YOCTO #10623]
+
+Upstream-Status: Backport [commit 45eca34c754d416a38bee90fb2d3c110a0b6cc5f]
+
+Signed-off-by: Patrick Ohly <patrick.o...@intel.com>
+---
+ pseudo_client.c | 2 +-
+ 1 file changed, 1 insertion(+), 1 deletion(-)
+
+diff --git a/pseudo_client.c b/pseudo_client.c
+index 6a08df3..b1a00fa 100644
+--- a/pseudo_client.c
b/pseudo_client.c
+@@ -1676,7 +1676,7 @@ pseudo_client_op(pseudo_op_t op, int access, int fd, int
dirfd, const char *path
+* empty path for that.
+*/
+ if (path_extra_1) {
+- size_t full_len = path_extra_1len + 1 + pathlen;
++ size_t full_len = path_extra_1len + 1 + pathlen - strip_slash;
+ size_t partial_len = pathlen - 1 - strip_slash;
+ if (path_extra_2) {
+ full_len += path_extra_2len + 1;
+--
+2.1.4
+
diff --git a/meta/recipes-devtools/pseudo/pseudo_1.8.1.bb
b/meta/recipes-devtools/pseudo/pseudo_1.8.1.bb
index fb70034..90b53c0 100644
--- a/meta/recipes-devtools/pseudo/pseudo_1.8.1.bb
+++ b/meta/recipes-devtools/pseudo/pseudo_1.8.1.bb
@@ -10,6 +10,7 @@ SRC_URI =
"http://downloads.yoctoproject.org/releases/pseudo/${BPN}-${PV}.tar.bz
file://0001-Quiet-diagnostics-during-startup-for-pseudo-d.patch \
file://0002-Use-correct-file-descriptor.patch \
file://0003-Fix-renameat-parallel-to-previous-fix-to-rename.patch \
+ file://More-correctly-fix-xattrs.patch \
"
SRC_URI[md5sum] = "ee38e4fb62ff88ad067b1a5a3825bac7"
diff --git a/meta/recipes-devtools/pseudo/pseudo_git.bb
b/meta/recipes-devtools/pseudo/pseudo_git.bb
index 8110b1a..ac923bb 100644
--- a/meta/recipes-devtools/pseudo/pseudo_git.bb
+++ b/meta/recipes-devtools/pseudo/pseudo_git.bb
@@ -1,6 +1,6 @@
require pseudo.inc
-SRCREV = "befc6dbd6469d428c9e0830dbe51bdf7ac39d9ae"
+SRCREV = "45eca34c754d416a38bee90fb2d3c110a0b6cc5f"
PV = "1.8.1+git${SRCPV}"
DEFAULT_PREFERENCE = "-1"
--
2.1.4
--
___
Openembedded-core mailing list
Openembedded-core@lists.openembedded.org
http://lists.openembedded.org/mailman/listinfo/openembedded-core
[OE-core] [PATCH] pseudo: include fix for xattr corruption
pseudo_1.8.1.bb gets the backported patch and pseudo_git.bb gets
updated to include the commit, which currently is the most recent
commit on the pseudo master branch.
There's just one problem - it breaks other cases:
foo: security.SMACK64: No such attribute
Signed-off-by: Patrick Ohly <patrick.o...@intel.com>
---
.../pseudo/files/More-correctly-fix-xattrs.patch | 37 ++
meta/recipes-devtools/pseudo/pseudo_1.8.1.bb | 1 +
meta/recipes-devtools/pseudo/pseudo_git.bb | 2 +-
3 files changed, 39 insertions(+), 1 deletion(-)
create mode 100644
meta/recipes-devtools/pseudo/files/More-correctly-fix-xattrs.patch
diff --git a/meta/recipes-devtools/pseudo/files/More-correctly-fix-xattrs.patch
b/meta/recipes-devtools/pseudo/files/More-correctly-fix-xattrs.patch
new file mode 100644
index 000..3d178f9
--- /dev/null
+++ b/meta/recipes-devtools/pseudo/files/More-correctly-fix-xattrs.patch
@@ -0,0 +1,37 @@
+From 45eca34c754d416a38bee90fb2d3c110a0b6cc5f Mon Sep 17 00:00:00 2001
+From: Seebs <se...@seebs.net>
+Date: Thu, 3 Nov 2016 11:36:12 -0500
+Subject: [PATCH] More-correctly fix xattrs
+
+Fix provided by Patrick Ohly <patrick.o...@intel.com>. This resolves
+the actual cause of the path length mismatches, and explains why
+I couldn't quite explain why the previous one had only sometimes
+worked, also why it showed up on directories but not plain files.
+
+Signed-off-by: Seebs <se...@seebs.net>
+
+Fixes [YOCTO #10623]
+
+Upstream-Status: Backport [commit 45eca34c754d416a38bee90fb2d3c110a0b6cc5f]
+
+Signed-off-by: Patrick Ohly <patrick.o...@intel.com>
+---
+ pseudo_client.c | 2 +-
+ 1 file changed, 1 insertion(+), 1 deletion(-)
+
+diff --git a/pseudo_client.c b/pseudo_client.c
+index 6a08df3..b1a00fa 100644
+--- a/pseudo_client.c
b/pseudo_client.c
+@@ -1676,7 +1676,7 @@ pseudo_client_op(pseudo_op_t op, int access, int fd, int
dirfd, const char *path
+* empty path for that.
+*/
+ if (path_extra_1) {
+- size_t full_len = path_extra_1len + 1 + pathlen;
++ size_t full_len = path_extra_1len + 1 + pathlen - strip_slash;
+ size_t partial_len = pathlen - 1 - strip_slash;
+ if (path_extra_2) {
+ full_len += path_extra_2len + 1;
+--
+2.1.4
+
diff --git a/meta/recipes-devtools/pseudo/pseudo_1.8.1.bb
b/meta/recipes-devtools/pseudo/pseudo_1.8.1.bb
index fb70034..90b53c0 100644
--- a/meta/recipes-devtools/pseudo/pseudo_1.8.1.bb
+++ b/meta/recipes-devtools/pseudo/pseudo_1.8.1.bb
@@ -10,6 +10,7 @@ SRC_URI =
"http://downloads.yoctoproject.org/releases/pseudo/${BPN}-${PV}.tar.bz
file://0001-Quiet-diagnostics-during-startup-for-pseudo-d.patch \
file://0002-Use-correct-file-descriptor.patch \
file://0003-Fix-renameat-parallel-to-previous-fix-to-rename.patch \
+ file://More-correctly-fix-xattrs.patch \
"
SRC_URI[md5sum] = "ee38e4fb62ff88ad067b1a5a3825bac7"
diff --git a/meta/recipes-devtools/pseudo/pseudo_git.bb
b/meta/recipes-devtools/pseudo/pseudo_git.bb
index 8110b1a..ac923bb 100644
--- a/meta/recipes-devtools/pseudo/pseudo_git.bb
+++ b/meta/recipes-devtools/pseudo/pseudo_git.bb
@@ -1,6 +1,6 @@
require pseudo.inc
-SRCREV = "befc6dbd6469d428c9e0830dbe51bdf7ac39d9ae"
+SRCREV = "45eca34c754d416a38bee90fb2d3c110a0b6cc5f"
PV = "1.8.1+git${SRCPV}"
DEFAULT_PREFERENCE = "-1"
--
2.1.4
--
___
Openembedded-core mailing list
Openembedded-core@lists.openembedded.org
http://lists.openembedded.org/mailman/listinfo/openembedded-core
[OE-core] [PATCH] qemuboot.bbclass: do not hook into do_rootfs
Writing qemuboot.conf in write_qemuboot_conf() does not modify the
rootfs and thus conceptually shouldn't be executed as part of rootfs
creation.
Running it as separate task is cleaner and fixes the problem of
missing qemuboot.conf files for meta-swupd virtual images; those
images replace do_rootfs and ROOTFS_POSTPROCESS_COMMANDs don't run at
all.
The task gets added such that it runs roughly at the same time as
before. Probably it doesn't actually need to depend on do_rootfs, but
this way we don't write a useless qemuboot.conf in cases where
do_rootfs fails.
Signed-off-by: Patrick Ohly <patrick.o...@intel.com>
---
meta/classes/qemuboot.bbclass | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
diff --git a/meta/classes/qemuboot.bbclass b/meta/classes/qemuboot.bbclass
index 39df3ad..8b1d4d0 100644
--- a/meta/classes/qemuboot.bbclass
+++ b/meta/classes/qemuboot.bbclass
@@ -42,7 +42,7 @@ QB_DEFAULT_FSTYPE ?= "ext4"
QB_OPT_APPEND ?= "-show-cursor"
# Create qemuboot.conf
-ROOTFS_POSTPROCESS_COMMAND += "write_qemuboot_conf; "
+addtask do_write_qemuboot_conf after do_rootfs before do_image
def qemuboot_vars(d):
build_vars = ['MACHINE', 'TUNE_ARCH', 'DEPLOY_DIR_IMAGE',
@@ -51,8 +51,8 @@ def qemuboot_vars(d):
'STAGING_DIR_HOST']
return build_vars + [k for k in d.keys() if k.startswith('QB_')]
-write_qemuboot_conf[vardeps] += "${@' '.join(qemuboot_vars(d))}"
-python write_qemuboot_conf() {
+do_write_qemuboot_conf[vardeps] += "${@' '.join(qemuboot_vars(d))}"
+python do_write_qemuboot_conf() {
import configparser
qemuboot = "%s/%s.qemuboot.conf" % (d.getVar('DEPLOY_DIR_IMAGE', True),
d.getVar('IMAGE_NAME', True))
--
2.1.4
--
___
Openembedded-core mailing list
Openembedded-core@lists.openembedded.org
http://lists.openembedded.org/mailman/listinfo/openembedded-core
Re: [OE-core] [PATCH 2/2] base-passwd: set root's default password to 'root'
On Thu, 2016-11-24 at 11:38 +0800, Robert Yang wrote: > Currently, debug-tweaks is in EXTRA_IMAGE_FEATURES by default for poky, and > there is no passwd, so that user can login easily without a passwd, I think > that current status is more unsafe ? Both well-known password and no password are unsafe. User "root" with password "root" is not even "more" safe already now, because tools that brute-force logins try that. Choosing something else would be a bit safer for a short while until the tools add it to their dictionary. Poky is also targeting a different audience than OE-core. Poky can assume to be used in a secure environment, OE-core can't (because it might be used for all kinds of devices). -- Best Regards, Patrick Ohly The content of this message is my personal opinion only and although I am an employee of Intel, the statements I make here in no way represent Intel's position on the issue, nor am I authorized to speak on behalf of Intel on this matter. -- ___ Openembedded-core mailing list Openembedded-core@lists.openembedded.org http://lists.openembedded.org/mailman/listinfo/openembedded-core
Re: [OE-core] Contents of non-rootfs partitions
On Thu, 2016-11-24 at 07:15 +0100, Ulrich Ölmann wrote: > On Wed, Nov 23, 2016 at 04:56:56PM +0100, Patrick Ohly wrote: > > Excluding only the directory content but not the actual directory is > > indeed a good point. I'm a bit undecided. When excluding only the > > directory content, there's no way of building a rootfs without that > > mount point, if that's desired. OTOH, when excluding also the directory, > > the data would have to be staged under a different path in the rootfs > > and the mount point would have to be a separate, empty directory. > > > > I'm leaning towards excluding the directory content and keeping the > > directory. > > what about having both possibilities by leaning against the syntax that rsync > uses to specify if a whole source directory or only it's contents shall be > synced to some destination site (see [1])? I like that idea. -- Best Regards, Patrick Ohly The content of this message is my personal opinion only and although I am an employee of Intel, the statements I make here in no way represent Intel's position on the issue, nor am I authorized to speak on behalf of Intel on this matter. -- ___ Openembedded-core mailing list Openembedded-core@lists.openembedded.org http://lists.openembedded.org/mailman/listinfo/openembedded-core
Re: [OE-core] Contents of non-rootfs partitions
On Wed, 2016-11-23 at 15:22 +0200, Ed Bartosh wrote: > On Wed, Nov 23, 2016 at 02:08:28PM +0100, Kristian Amlie wrote: > > On 23/11/16 13:08, Ed Bartosh wrote: > > > On Tue, Nov 22, 2016 at 12:54:52PM +0100, Kristian Amlie wrote: > > >> On 22/11/16 12:10, Patrick Ohly wrote: > > >>>> ... > > >>> > > >>> All of these introduce some special mechanism. Let me propose something > > >>> that might integrate better with the existing tooling: > > >>> > > >>> The "rootfs" directory gets redefined as representing the entire virtual > > >>> file system. When creating a disk image, it gets split up into different > > >>> partitions based on the image configuration. > > >>> > > >>> For example, the /home or /data directories in the rootfs could hold the > > >>> content that in some image configurations goes into separate partitions. > > >>> > > >>> The advantage of this approach is that the tooling for staging content > > >>> for image creation does not need to be changed. The same staged content > > >>> then can be used to create different images, potentially even using > > >>> different partition layouts. > > >> > > >> That's a very good idea. I think it beats all of my suggestions! > > >> > > >>> To implement this approach with wic, wic needs to be taught how to > > >>> exclude directories from the main rootfs. Ideally, the mkfs.* tools > > >>> should also support that without having to make an intermediate copy of > > >>> the files for a certain partition, but initially wic could create > > >>> temporary directory trees. > > >> > > >> Yes, some work would be needed here, but ultimately it would be > > >> contained within wic and related tools, which is a good thing. > > >> > > > > > > I support the idea. Let's discuss the details of implementation and > > > create a bug in bugzilla to track the development > > > > Do you want me to create the ticket? (it'll be my first, so apologies in > > advance if I omit something important) > > > > > This can be done by extending existing rootfs plugin. It should be able > > > to do 2 things: > > > > > > - populate content of one rootfs directory to the partition. We can > > > extend syntax of --rootfs-dir parameter to specify optional directory > > > path to use > > > > > > - exclude rootfs directories when populating partitions. I'd propose to > > > introduce --exclude-dirs wks parser option to handle this. > > > > > > Example of wks file with proposed new options: > > > part / --source rootfs --rootfs-dir=core-image-minimal --ondisk > > > sda --fstype=ext4 --label root --align 1024 --exclude-dirs data > > > --exclude-dirs home > > > part /data --source rootfs --rootfs-dir=core-image-minimal:/home --ondisk > > > sda --fstype=ext4 --label data --align 1024 > > > part /home --source rootfs --rootfs-dir=core-image-minimal:/data --ondisk > > > sda --fstype=ext4 --label data --align 1024 > > > > > > Does this make sense? > > > > Looks good. The only thing I would question is that, in the interest of > > reducing redundancy, maybe we should omit --exclude-dirs and have wic > > figure this out by combining all the entries, since "--exclude-dirs > > " and the corresponding "part " will almost always come in > > pairs. Possibly we could mark the "/" partition with one single > > --no-overlapping-dirs to force wic to make this consideration. Or do you > > think that's too magical? > > > Tt's quite implicit from my point of view. However, if people like it we > can implement it this way. I prefer the explicit --exclude-dirs. It's less surprising and perhaps there are usages for having the same content in different partitions (redundancy, factory reset, etc.). Excluding only the directory content but not the actual directory is indeed a good point. I'm a bit undecided. When excluding only the directory content, there's no way of building a rootfs without that mount point, if that's desired. OTOH, when excluding also the directory, the data would have to be staged under a different path in the rootfs and the mount point would have to be a separate, empty directory. I'm leaning towards excluding the directory content and keeping the directory. -- Best Regards, Patrick Ohly The content of this message is my personal opinion only and although I am an employee of Intel, the statements I make here in no way represent Intel's position on the issue, nor am I authorized to speak on behalf of Intel on this matter. -- ___ Openembedded-core mailing list Openembedded-core@lists.openembedded.org http://lists.openembedded.org/mailman/listinfo/openembedded-core
Re: [OE-core] [PATCH 2/2] base-passwd: set root's default password to 'root'
On Tue, 2016-11-22 at 23:49 -0800, Robert Yang wrote: > [YOCTO #10710] > > Otherwise, we can't login as root when debug-tweaks is not in > IMAGE_FEATURES, and there is no other users to login by default, so > there is no way to login. Wait a second, are you really suggesting that OE-core should have a default root password in its default configuration? That's very bad practice and I'm against doing it this way. Having a default password is one of the common vulnerabilities in actual devices on the market today. OE-core should make it hard to make that mistake, not actively introduce it. So if you think that having a root password set (instead of empty), then at least make it an opt-in behavior that explicitly has to be selected. Make it an image feature so that images with and without default password can be build in the same build configuration. Changing base-passwd doesn't achieve that. Even then I'm still wondering what the benefit of a well-known password compared to no password is. Both are equally insecure, so someone who wants to allow logins might as well go with "empty password". -- Best Regards, Patrick Ohly The content of this message is my personal opinion only and although I am an employee of Intel, the statements I make here in no way represent Intel's position on the issue, nor am I authorized to speak on behalf of Intel on this matter. -- ___ Openembedded-core mailing list Openembedded-core@lists.openembedded.org http://lists.openembedded.org/mailman/listinfo/openembedded-core
Re: [OE-core] Contents of non-rootfs partitions
On Tue, 2016-11-22 at 10:05 +0100, Kristian Amlie wrote: > Hey > > In Mender we are using a dual rootfs partition layout (A/B layout) > with a persistent data partition on the side. We're using wic to do > the actual image building. > > However, putting files into this data partition is not a > straightforward operation ATM. All recipes in OE put files into the > rootfs only, and we'd like there to be a way to specify that files > should go into the data partition. > > I haven't seen any straightforward way to do this in OE. Some > suggestions for how it could be done: > > * A "DATA_FILES" variable (like "IMAGE_BOOT_FILES"), where you can > list files you'd like to be included. This is very straightforward, > but has the disadvantage of being a bit hairy when paths are involved, > and it doesn't allow for much logic. > > * Keep a special "data" directory, something similar to sysroot, where > recipes could install files into in order to have it included on the > data partition. We could potentially have a bbclass for use in recipes > to make this more streamlined. > > * Have a special recipe that users can bbappend, which specifically > install to the data partition (this might also need a "data" > directory, like above). All of these introduce some special mechanism. Let me propose something that might integrate better with the existing tooling: The "rootfs" directory gets redefined as representing the entire virtual file system. When creating a disk image, it gets split up into different partitions based on the image configuration. For example, the /home or /data directories in the rootfs could hold the content that in some image configurations goes into separate partitions. The advantage of this approach is that the tooling for staging content for image creation does not need to be changed. The same staged content then can be used to create different images, potentially even using different partition layouts. To implement this approach with wic, wic needs to be taught how to exclude directories from the main rootfs. Ideally, the mkfs.* tools should also support that without having to make an intermediate copy of the files for a certain partition, but initially wic could create temporary directory trees. -- Best Regards, Patrick Ohly The content of this message is my personal opinion only and although I am an employee of Intel, the statements I make here in no way represent Intel's position on the issue, nor am I authorized to speak on behalf of Intel on this matter. -- ___ Openembedded-core mailing list Openembedded-core@lists.openembedded.org http://lists.openembedded.org/mailman/listinfo/openembedded-core
Re: [OE-core] [PATCH] libpcap: Fix build when PACKAGECONFIG ipv6 is not enable
On Thu, 2016-11-17 at 09:24 -0700, Christopher Larson wrote: > > On Thu, Nov 17, 2016 at 9:21 AM, Fabio Berton > <fabio.ber...@ossystems.com.br> wrote: > No, I created a patch, git format-patch and then edit > generated files with Upstream-Status tag and added to recipe. > Is this wrong? > > As I indicated in my first reply, it’s best to put the tag outside the > generated patch (above it, or below the —-), as it isn’t part of the > commit, only part of the patch file. Now I'm confused. My understanding was that http://www.openembedded.org/wiki/Commit_Patch_Message_Guidelines#Patch_Header_Recommendations explicitly asks for Upstream-Status in the patch header. Taking an existing example, is http://cgit.openembedded.org/openembedded-core/tree/meta/recipes-core/systemd/systemd/0001-core-device.c-Change-the-default-device-timeout-to-2.patch doing it wrong? > It’s minor, and you don’t need to re-submit, but in general the tag > is not part of the commit message. For example, if your patch was > applied to a git repository with git-am, it’d be in the commit > message, which should not be the case. Yes, that's indeed the effect. That has pros (the Upstream-Status tag is preserved when working with devtool) and cons (patch as attached to a recipe is not the same as the patch upstream). -- Best Regards, Patrick Ohly The content of this message is my personal opinion only and although I am an employee of Intel, the statements I make here in no way represent Intel's position on the issue, nor am I authorized to speak on behalf of Intel on this matter. -- ___ Openembedded-core mailing list Openembedded-core@lists.openembedded.org http://lists.openembedded.org/mailman/listinfo/openembedded-core
Re: [OE-core] Modifying SRC_URI from anonymous python
On Thu, 2016-11-17 at 13:28 -0800, Andre McCurdy wrote:
> I have a supplier who provides recipes which set SRC_URI to their
> private git servers. To make those same recipes usable by others (ie
> me), some anonymous python is used to transform the default SRC_URI
> (elements which contain private git URLs are replaced, patches and
> other files are left as-is).
>
> This apparently worked in OE 2.0 but from 2.1 onwards the anonymous
> python which modifies SRC_URI races with the anonymous python in
> base.bbclass which parses SRC_URI to determine additional
> do_fetch/do_unpack dependencies and whether or not to call
> fetch2.get_srcrev().
I have a patch which I intend to submit to bitbake which adds priorities
for anonymous python methods:
https://github.com/pohly/poky/commit/9d959e50e5f27d149654f5db0aff2fe51365ad68
With that in place, the anonymous python method which transforms the
SRC_URI could request to run sooner than the base.bbclass anonymous
python.
However, I am not sure whether that should be backported to 2.1 and 2.2.
> Specifically I get failures because
> fetch2.get_srcrev() sees the original SRC_URI and tries to resolve
> AUTOREV from a repo to which I don't have access.
>
> The proposed solution from the supplier is this patch to base.bbclass:
>
> @@ -598,7 +598,7 @@ python () {
> d.appendVarFlag('do_unpack', 'depends', '
> file-native:do_populate_sysroot')
>
> if needsrcrev:
> -d.setVar("SRCPV", "${@bb.fetch2.get_srcrev(d)}")
> +d.setVar("SRCPV", "${@bb.fetch2.get_srcrev(d)}", parsing=True)
>
> set_packagetriplet(d)
>
> After reading the setVar source I'm not very clear how or why this
> works, but it looks dubious. What is parsing=True intended to do?
>
> Is it documented somewhere that modifying SRC_URI from anonymous
> python isn't allowed? I've now seen two suppliers both independently
> run into the same problem when updating to OE 2.1.
Sorry, I don't know about that.
--
Best Regards, Patrick Ohly
The content of this message is my personal opinion only and although
I am an employee of Intel, the statements I make here in no way
represent Intel's position on the issue, nor am I authorized to speak
on behalf of Intel on this matter.
--
___
Openembedded-core mailing list
Openembedded-core@lists.openembedded.org
http://lists.openembedded.org/mailman/listinfo/openembedded-core
[OE-core] truncating quoted text in emails
Hello! While trying to follow patch reviews on this list I noticed that (subjectively?) quite a few replies completely quote the original email and then just add a few lines. That looks fine in GMail where the quoted text is folded, but not so much in mail readers where quoted text is unfolded (Evolution). A random example: http://lists.openembedded.org/pipermail/openembedded-core/2016-November/128861.html Not sure what the netiquette is for this list, but at least I would appreciate a bit more aggressive trimming in replies - thanks! ;-} -- Best Regards, Patrick Ohly The content of this message is my personal opinion only and although I am an employee of Intel, the statements I make here in no way represent Intel's position on the issue, nor am I authorized to speak on behalf of Intel on this matter. -- ___ Openembedded-core mailing list Openembedded-core@lists.openembedded.org http://lists.openembedded.org/mailman/listinfo/openembedded-core
Re: [OE-core] [PATCH] kernel.bbclass: Allow ${S} to be overridden
On Fri, 2016-11-11 at 19:39 +, Paul Barker wrote:
> On Wed, 9 Nov 2016 15:42:29 -0800
> Andre McCurdy <armccu...@gmail.com> wrote:
>
> > The solution I use (learned the hard way) is to ensure that "S = ..."
> > comes after "inherit kernel" in the kernel recipe.
> >
>
> Ok, yea, this works for me.
>
> Is this expected? S is set to ${STAGING_KERNEL_DIR} at the start of
> kernel.bbclass. The base_do_unpack_append() function is defined later
> in kernel.bbclass. If S is set to ${WORKDIR}/${BP} only after
> kernel.bbclass is inherited, then that's after base_do_unpack_append()
> has been parsed.
Parsing itself does not expand variable references (*), so it is
possible to use a variable that gets changed later on and use the value
that the variable eventually ends up having.
(*) There's one exception - the := assignment expands based on the
currently defined variables at the time of parsing the assignment.
--
Best Regards, Patrick Ohly
The content of this message is my personal opinion only and although
I am an employee of Intel, the statements I make here in no way
represent Intel's position on the issue, nor am I authorized to speak
on behalf of Intel on this matter.
--
___
Openembedded-core mailing list
Openembedded-core@lists.openembedded.org
http://lists.openembedded.org/mailman/listinfo/openembedded-core
Re: [OE-core] [PATCH V2] sqlite3: Revert ad601c7962 from 3.14.1 amalgamation package
On Thu, 2016-11-10 at 18:53 +0100, Patrick Ohly wrote: > On Thu, 2016-10-13 at 13:16 -0700, Jianxun Zhang wrote: > > It turns out this change between 3.12.2 and 3.13 introduces > > a 2% increase of build time based on statistic data in > > bz10367. > > Let me add that this patch increased build performance in Ostro even > more: apparently one big impact of the sqlite performance issue is on > pseudo. Ostro depends fairly heavily on pseudo because of meta-swupd and > xattrs on all files. > > When this patch and others recently landed in Ostro, total build times > dropped from 4:46h (build #508, > https://ostroproject.org/jenkins/job/build_intel-corei7-64/2763/console) > to 2:07h (build #510, > https://ostroproject.org/jenkins/job/build_intel-corei7-64/2831/console). In case that someone wonders: in addition to that enhancement, we can still do even better by also reducing the work that meta-swupd causes. A build with my recent meta-swupd enhancements takes the build time down to 1:05h. -- Best Regards, Patrick Ohly The content of this message is my personal opinion only and although I am an employee of Intel, the statements I make here in no way represent Intel's position on the issue, nor am I authorized to speak on behalf of Intel on this matter. -- ___ Openembedded-core mailing list Openembedded-core@lists.openembedded.org http://lists.openembedded.org/mailman/listinfo/openembedded-core
Re: [OE-core] [PATCH V2] sqlite3: Revert ad601c7962 from 3.14.1 amalgamation package
On Thu, 2016-10-13 at 13:16 -0700, Jianxun Zhang wrote: > It turns out this change between 3.12.2 and 3.13 introduces > a 2% increase of build time based on statistic data in > bz10367. Let me add that this patch increased build performance in Ostro even more: apparently one big impact of the sqlite performance issue is on pseudo. Ostro depends fairly heavily on pseudo because of meta-swupd and xattrs on all files. When this patch and others recently landed in Ostro, total build times dropped from 4:46h (build #508, https://ostroproject.org/jenkins/job/build_intel-corei7-64/2763/console) to 2:07h (build #510, https://ostroproject.org/jenkins/job/build_intel-corei7-64/2831/console). That could also be because of other improvements, perhaps even in our CI hardware, so take those numbers with a large chunk of salt. However, in local builds with and without this patch (i.e. everything else the same) I also see big differences for pseudo-heavy operations: $ buildstats-diff --diff-attr walltime --min-val 60 with-patch/ without-patch/ Ignoring tasks less than 01:00.0 (60.0s) Ignoring differences less than 00:02.0 (2.0s) PKG TASK ABSDIFF RELDIFF WALLTIME1 -> WALLTIME2 ... bundle-ostro-image-swupd-qa-bundle-bdo_rootfs 78.1s +115.4% 67.7s -> 145.8s bundle-ostro-image-swupd-qa-bundle-ado_rootfs 80.3s +116.8% 68.8s -> 149.1s bundle-ostro-image-swupd-qa-bundle-ado_image 106.8s +291.9% 36.6s -> 143.3s bundle-ostro-image-swupd-qa-bundle-bdo_image 107.9s +298.2% 36.2s -> 144.1s bundle-ostro-image-swupd-mega do_image 244.4s +74.2% 329.2s -> 573.6s bundle-ostro-image-swupd-world-dev do_rootfs 246.7s +207.2% 119.1s -> 365.8s bundle-ostro-image-swupd-world-dev do_image 269.2s +83.5% 322.6s -> 591.7s bundle-ostro-image-swupd-mega do_rootfs 272.6s +246.1% 110.8s -> 383.3s ostro-image-swupd do_rootfs 676.1s +808.1% 83.7s -> 759.8s bundle-ostro-image-swupd-world-dev do_copy_bundle_contents 1339.5s +2957.6% 45.3s -> 1384.8s bundle-ostro-image-swupd-qa-bundle-bdo_copy_bundle_contents 1475.0s +3147.8% 46.9s -> 1521.9s bundle-ostro-image-swupd-qa-bundle-ado_copy_bundle_contents 1503.9s +3283.0% 45.8s -> 1549.8s Cumulative walltime: 6070.9s+326.9%30:57.3 (1857.3s) -> 2:12:08.2 (7928.2s) So it really seems that the sqlite change is a very relevant improvement. That leads me to a bigger question: has upstream been notified about this? Our observation may also be relevant to other sqlite users. Besides, not getting this fixed upstream means that we'll have to do the same tricky revert for the next upstream version update. -- Best Regards, Patrick Ohly The content of this message is my personal opinion only and although I am an employee of Intel, the statements I make here in no way represent Intel's position on the issue, nor am I authorized to speak on behalf of Intel on this matter. -- ___ Openembedded-core mailing list Openembedded-core@lists.openembedded.org http://lists.openembedded.org/mailman/listinfo/openembedded-core
Re: [OE-core] using devtool for rebasing patches
On Thu, 2016-11-03 at 10:14 -0700, Christopher Larson wrote: > > On Thu, Nov 3, 2016 at 9:35 AM, Alexander Kanavin > <alexander.kana...@linux.intel.com> wrote: > > What you need to do here is to use git am --reject, which > would partially apply the patch, then find all the *.rej files > and let the user manually edit them in, then complete the > patch application with git am --continue. > > > With -3, it’ll merge it into the file with conflict markers, which is > a lot more pleasant to deal with than .rej files, even without the > merge-base being available. That's also what I would prefer. To deal with recipes that have something other than git as source one could create a local git repo with a branch that contains two commits (old version and new version), apply the patches on a second branch on top of the old version, then do the normal rebase with conflictstyle = diff3. At least I find that configstyle useful and have it in my ~/.gitconfig. -- Best Regards, Patrick Ohly The content of this message is my personal opinion only and although I am an employee of Intel, the statements I make here in no way represent Intel's position on the issue, nor am I authorized to speak on behalf of Intel on this matter. -- ___ Openembedded-core mailing list Openembedded-core@lists.openembedded.org http://lists.openembedded.org/mailman/listinfo/openembedded-core
Re: [OE-core] using devtool for rebasing patches
On Thu, 2016-11-03 at 18:01 +0200, Alexander Kanavin wrote: > Hello Paul, > > it would be very useful if devtool offered support for rebasing source > patches when they fail to apply +1 for that. I tried it the other way around: first "devtool modify", then rebasing using git (which only works when the recipe fetches from a git repo), and finally changing the recipe such that it refers to the version I rebased onto. But that version change in the recipe confused "devtool update-recipe" considerably (as in "added all patches between the version", if I remember correctly). -- Best Regards, Patrick Ohly The content of this message is my personal opinion only and although I am an employee of Intel, the statements I make here in no way represent Intel's position on the issue, nor am I authorized to speak on behalf of Intel on this matter. -- ___ Openembedded-core mailing list Openembedded-core@lists.openembedded.org http://lists.openembedded.org/mailman/listinfo/openembedded-core
[OE-core] [PATCH 2/2] archive.bbclass: fix do_ar_original archiving of multiple source repos
When a recipe uses more than one source which isn't a plain file (for
example, multiple git repos), then do_ar_original created the source
archives using the same filename and thus only archived one source.
The "name" parameter is used as file suffix to create unique names for
each source, leading to archives following this pattern:
deploy/${TARGET_SYS}/${PF}/${PF}[-].tar.gz.
The ${PF} part is a bit redundant, which may or may not be
desirable. The patch is more localized this way (no need to modify
create_tarball()).
For example, meta-oic's iotivity_1.1.1.bb uses:
url_iotivity = "git://github.com/iotivity/iotivity.git"
branch_iotivity = "1.1-rel"
SRC_URI =
"${url_iotivity};destsuffix=${S};branch=${branch_iotivity};protocol=http;"
url_tinycbor = "git://github.com/01org/tinycbor.git"
SRC_URI +=
"${url_tinycbor};name=tinycbor;destsuffix=${S}/extlibs/tinycbor/tinycbor;protocol=http"
url_hippomocks = "git://github.com/dascandy/hippomocks.git"
SRC_URI +=
"${url_hippomocks};name=hippomocks;destsuffix=${S}/extlibs/hippomocks-master;protocol=http"
SRC_URI += "file://hippomocks_mips_patch"
url_gtest =
"http://pkgs.fedoraproject.org/repo/pkgs/gtest/gtest-1.7.0.zip/2d6ec8ccdf5c46b05ba54a9fd1d130d7/gtest-1.7.0.zip;
SRC_URI += "${url_gtest};name=gtest;subdir=${BP}/extlibs/gtest"
url_sqlite = "http://www.sqlite.org/2015/sqlite-amalgamation-3081101.zip;
SRC_URI +=
"${url_sqlite};name=sqlite3;subdir=${BP}/extlibs/sqlite3;unpack=false"
These now get archived in deploy/sources/*/iotivity-1.1.1-r2/ as:
gtest-1.7.0.zip iotivity-1.1.1-r2-recipe.tar.gz
sqlite-amalgamation-3081101.zip
hippomocks_mips_patch iotivity-1.1.1-r2.tar.gz
iotivity-1.1.1-r2-hippomocks.tar.gz iotivity-1.1.1-r2-tinycbor.tar.gz
Signed-off-by: Patrick Ohly <patrick.o...@intel.com>
---
meta/classes/archiver.bbclass | 17 -
1 file changed, 16 insertions(+), 1 deletion(-)
diff --git a/meta/classes/archiver.bbclass b/meta/classes/archiver.bbclass
index 5f9c91d..1d8e863 100644
--- a/meta/classes/archiver.bbclass
+++ b/meta/classes/archiver.bbclass
@@ -151,6 +151,7 @@ python do_ar_original() {
encoded = bb.fetch2.encodeurl(decoded)
urls[i] = encoded
fetch = bb.fetch2.Fetch(urls, d)
+tarball_suffix = {}
for url in fetch.urls:
local = fetch.localpath(url).rstrip("/");
if os.path.isfile(local):
@@ -158,7 +159,21 @@ python do_ar_original() {
elif os.path.isdir(local):
tmpdir = tempfile.mkdtemp(dir=d.getVar('ARCHIVER_WORKDIR', True))
fetch.unpack(tmpdir, (url,))
-create_tarball(d, tmpdir + '/.', '', ar_outdir)
+# To handle recipes with more than one source, we add the "name"
+# URL parameter as suffix. We treat it as an error when
+# there's more than one URL without a name, or a name gets reused.
+# This is an additional safety net, in practice the name has
+# to be set when using the git fetcher, otherwise SRCREV cannot
+# be set separately for each URL.
+params = bb.fetch2.decodeurl(url)[5]
+name = params.get('name', '')
+if name in tarball_suffix:
+if not name:
+bb.fatal("Cannot determine archive names for original
source because 'name' URL parameter is unset in more than one URL. Add it to at
least one of these: %s %s" % (tarball_suffix[name], url))
+else:
+bb.fatal("Cannot determine archive names for original
source because 'name=' URL parameter '%s' is used twice. Make it unique in: %s
%s" % (tarball_suffix[name], url))
+tarball_suffix[name] = url
+create_tarball(d, tmpdir + '/.', name, ar_outdir)
# Emit patch series files for 'original'
bb.note('Writing patch series files...')
--
2.1.4
--
___
Openembedded-core mailing list
Openembedded-core@lists.openembedded.org
http://lists.openembedded.org/mailman/listinfo/openembedded-core
[OE-core] [PATCH 0/2] fix do_ar_original (multiple git sources, absolute path)
meta-oic's iotivity_1.1.1.bb recently started using destsuffix=${S}
for different git source URLs. This breaks randomly (depending on task
scheduling) when archive.bbclass and ARCHIVER_MODE[src] = "original"
are active, because the archiver then unpacks again into ${S} in
parallel to the running build.
While fixing that it was noticed that the archiver also did not
archive all upstream sources because it used the same .tar.gz filename
for all git repos.
The following changes since commit 1e13d1fd22186af5544e7248dc12635cd2f2e08b:
meta-environment: ensure corret TOOLCHAIN_CONFIGSITE_NOCACHE value
(2016-09-26 10:05:53 +0100)
are available in the git repository at:
git://github.com/pohly/openembedded-core do_ar_original
https://github.com/pohly/openembedded-core/tree/do_ar_original
Patrick Ohly (2):
archiver.bbclass: ignore unpack sub-directories in do_ar_original
archive.bbclass: fix do_ar_original archiving of multiple source repos
meta/classes/archiver.bbclass | 37 +++--
1 file changed, 35 insertions(+), 2 deletions(-)
--
2.1.4
--
___
Openembedded-core mailing list
Openembedded-core@lists.openembedded.org
http://lists.openembedded.org/mailman/listinfo/openembedded-core
[OE-core] [PATCH 1/2] archiver.bbclass: ignore unpack sub-directories in do_ar_original
Support for absolute paths in the "subdir" parameter was recently
added (bitbake rev: c3873346c6fa). The git fetcher has supported
absolute paths in "destsuffix" already before.
When the path is absolute as in destsuffix=${S}/foobar, the tmpdir
used by do_ar_original gets ignored, which breaks:
- source code archiving (tmpdir is empty)
- compilation due to race conditions (for example, ${S} getting
modified by do_ar_original while do_compile runs)
To solve this, these parameters get removed from URLs before
instantiating the fetcher for them.
This is done unconditionally also for relative paths, because these
paths are not useful when archiving the original source (upstream
source does not have them, they only get used by the recipe during
compilation).
Signed-off-by: Patrick Ohly <patrick.o...@intel.com>
---
meta/classes/archiver.bbclass | 20 +++-
1 file changed, 19 insertions(+), 1 deletion(-)
diff --git a/meta/classes/archiver.bbclass b/meta/classes/archiver.bbclass
index 2f3b278..5f9c91d 100644
--- a/meta/classes/archiver.bbclass
+++ b/meta/classes/archiver.bbclass
@@ -132,7 +132,25 @@ python do_ar_original() {
ar_outdir = d.getVar('ARCHIVER_OUTDIR', True)
bb.note('Archiving the original source...')
-fetch = bb.fetch2.Fetch([], d)
+urls = d.getVar("SRC_URI", True).split()
+# destsuffix (git fetcher) and subdir (everything else) are allowed to be
+# absolute paths (for example, destsuffix=${S}/foobar).
+# That messes with unpacking inside our tmpdir below, because the fetchers
+# will then unpack in that directory and completely ignore the tmpdir.
+# That breaks parallel tasks relying on ${S}, like do_compile.
+#
+# To solve this, we remove these parameters from all URLs.
+# We do this even for relative paths because it makes the content of the
+# archives more useful (no extra paths that are only used during
+# compilation).
+for i, url in enumerate(urls):
+decoded = bb.fetch2.decodeurl(url)
+for param in ('destsuffix', 'subdir'):
+if param in decoded[5]:
+del decoded[5][param]
+encoded = bb.fetch2.encodeurl(decoded)
+urls[i] = encoded
+fetch = bb.fetch2.Fetch(urls, d)
for url in fetch.urls:
local = fetch.localpath(url).rstrip("/");
if os.path.isfile(local):
--
2.1.4
--
___
Openembedded-core mailing list
Openembedded-core@lists.openembedded.org
http://lists.openembedded.org/mailman/listinfo/openembedded-core
Re: [OE-core] [master][PATCH] openssl: security fix CVE-2016-6304
[resending from my Intel account, the one on GMX isn't subscribed] On Fri, 2016-09-23 at 21:06 +1200, Paul Eggleton wrote: > On Fri, 23 Sep 2016 11:56:41 Maxin B. John wrote: > > On Fri, Sep 23, 2016 at 04:48:37PM +0800, Anuj Mittal wrote: > > > Reference: > > > https://www.openssl.org/news/secadv/20160922.txt > > > > > > Upstream fix: > > > https://github.com/openssl/openssl/commit/e408c09bbf7c3057bda4b8d20bec1b3a > > > 7771c15b > > > > > > Signed-off-by: Anuj Mittal <anujx.mit...@intel.com> > > > --- > > > > > > .../openssl/openssl/CVE-2016-6304.patch| 75 > > > ++ > > Mid air collision with Patrick's patch. > > I guess for krogoth and jethro we have the choice of applying just this fix > or > the upgrade. Looking over the commits for 1.0.2i it does look like quite a > lot > more than the list of CVEs in the recent security advisory were fixed, and > it's somewhat concerning that the 1.0.2i release went out with an apparently > compile-breaking typo in it (subsequently fixed, patch applied in Patrick's > upgrade). The compile error is inside an #ifdef, so it could be that just that particular configuration hadn't been tested. But yes, one has to wonder. So what's preferred for OE-core master and the 2.2 release? Updating to 1.0.2i or backporting the critical patch? I don't have any strong opinion either way myself. -- Best Regards, Patrick Ohly The content of this message is my personal opinion only and although I am an employee of Intel, the statements I make here in no way represent Intel's position on the issue, nor am I authorized to speak on behalf of Intel on this matter. -- ___ Openembedded-core mailing list Openembedded-core@lists.openembedded.org http://lists.openembedded.org/mailman/listinfo/openembedded-core
Re: [OE-core] [PATCH 1/1] toolchain-shar-relocate.sh: Add error-handling
On Fri, 2016-09-23 at 15:48 +0100, Richard Purdie wrote: > On Wed, 2016-09-21 at 19:06 -0700, Christopher Larson wrote: > > > > On Wed, Sep 21, 2016 at 12:54 AM, <mariano.lo...@linux.intel.com> > > wrote: > > > diff --git a/meta/files/toolchain-shar-relocate.sh > > > b/meta/files/toolchain-shar-relocate.sh > > > index d4bcf0e..e491153 100644 > > > --- a/meta/files/toolchain-shar-relocate.sh > > > +++ b/meta/files/toolchain-shar-relocate.sh > > > @@ -1,3 +1,8 @@ > > > +if ! xargs --version > /dev/null 2>&1; then > > > + echo "xargs is required by the relocation script, please > > > install it firts. Abort!" > > > + exit 1 > > > +fi > > > > > This is pointless, xargs is mandated by posix/sus. > > It appears we have some users with stripped down containers where it > hasn't been installed though and having a user readable message is > better ... and even better is one without typos ;-} s/firts/first/ -- Best Regards, Patrick Ohly The content of this message is my personal opinion only and although I am an employee of Intel, the statements I make here in no way represent Intel's position on the issue, nor am I authorized to speak on behalf of Intel on this matter. -- ___ Openembedded-core mailing list Openembedded-core@lists.openembedded.org http://lists.openembedded.org/mailman/listinfo/openembedded-core
Re: [OE-core] [PATCH 1/2] openssl.inc: avoid random ptest failures
Hello! This patch can be applied stand-alone. I had it in a branch together with the 1.0.2i update and then didn't export it independently, hence the erroneous "PATCH 1/2" in the subject. -- Best Regards, Patrick Ohly The content of this message is my personal opinion only and although I am an employee of Intel, the statements I make here in no way represent Intel's position on the issue, nor am I authorized to speak on behalf of Intel on this matter. -- ___ Openembedded-core mailing list Openembedded-core@lists.openembedded.org http://lists.openembedded.org/mailman/listinfo/openembedded-core
[OE-core] [PATCH 1/2] openssl.inc: avoid random ptest failures
"make alltests" is sensitive to the timestamps of the installed
files. Depending on the order in which cp copies files, .o and/or
executables may end up with time stamps older than the source files.
Running tests then triggers recompilation attempts, which typically
will fail because dev tools and files are not installed.
"cp -a" is not enough because the files also have to be newer than
the installed header files. Setting the file time stamps to
the current time explicitly after copying solves the problem because
do_install_ptest_base is guaranteed to run after do_install.
Signed-off-by: Patrick Ohly <patrick.o...@intel.com>
---
meta/recipes-connectivity/openssl/openssl.inc | 13 +
1 file changed, 13 insertions(+)
diff --git a/meta/recipes-connectivity/openssl/openssl.inc
b/meta/recipes-connectivity/openssl/openssl.inc
index f83664c..a632d8a 100644
--- a/meta/recipes-connectivity/openssl/openssl.inc
+++ b/meta/recipes-connectivity/openssl/openssl.inc
@@ -215,6 +215,19 @@ do_install_ptest () {
mkdir -p ${D}${PTEST_PATH}/util
install util/opensslwrap.sh${D}${PTEST_PATH}/util
install util/shlib_wrap.sh ${D}${PTEST_PATH}/util
+ # Time stamps are relevant for "make alltests", otherwise
+ # make may try to recompile binaries. Not only must the
+ # binary files be newer than the sources, they also must
+ # be more recent than the header files in /usr/include.
+ #
+ # Using "cp -a" is not sufficient, because do_install
+ # does not preserve the original time stamps.
+ #
+ # So instead of using the original file stamps, we set
+ # the current time for all files. Binaries will get
+ # modified again later when stripping them, but that's okay.
+ touch ${D}${PTEST_PATH}
+ find ${D}${PTEST_PATH} -type f -print0 | xargs --verbose -0 touch -r
${D}${PTEST_PATH}
}
do_install_append_class-native() {
--
2.1.4
--
___
Openembedded-core mailing list
Openembedded-core@lists.openembedded.org
http://lists.openembedded.org/mailman/listinfo/openembedded-core
[OE-core] [PATCHv2] openssl: update to 1.0.2i (CVE-2016-6304 and more)
This update fixes several CVEs:
* OCSP Status Request extension unbounded memory growth (CVE-2016-6304)
* SWEET32 Mitigation (CVE-2016-2183)
* OOB write in MDC2_Update() (CVE-2016-6303)
* Malformed SHA512 ticket DoS (CVE-2016-6302)
* OOB write in BN_bn2dec() (CVE-2016-2182)
* OOB read in TS_OBJ_print_bio() (CVE-2016-2180)
* DTLS buffered message DoS (CVE-2016-2179)
* DTLS replay protection DoS (CVE-2016-2181)
* Certificate message OOB reads (CVE-2016-6306)
Of these, only CVE-2016-6304 is considered of high
severity. Everything else is low. CVE-2016-2177 and CVE-2016-2178 were
already fixed via local patches, which can be removed now.
See https://www.openssl.org/news/secadv/20160922.txt for details.
Some patches had to be refreshed and one compile error fix from
upstream's OpenSSL_1_0_2-stable was required. The server.pem
file is needed for test_dtls.
Signed-off-by: Patrick Ohly <patrick.o...@intel.com>
---
Notes:
Changes since v1:
* Install server.pem to get all ptests to pass.
meta/recipes-connectivity/openssl/openssl.inc | 1 +
.../openssl/openssl/CVE-2016-2177.patch| 286 -
.../openssl/openssl/CVE-2016-2178.patch| 51
.../openssl/Fix-typo-introduced-by-a03f81f4.patch | 29 +++
.../openssl/openssl/debian/ca.patch| 2 +-
.../openssl/openssl/parallel.patch | 17 +-
.../recipes-connectivity/openssl/openssl_1.0.2h.bb | 60 -
.../recipes-connectivity/openssl/openssl_1.0.2i.bb | 59 +
8 files changed, 104 insertions(+), 401 deletions(-)
delete mode 100644
meta/recipes-connectivity/openssl/openssl/CVE-2016-2177.patch
delete mode 100644
meta/recipes-connectivity/openssl/openssl/CVE-2016-2178.patch
create mode 100644
meta/recipes-connectivity/openssl/openssl/Fix-typo-introduced-by-a03f81f4.patch
delete mode 100644 meta/recipes-connectivity/openssl/openssl_1.0.2h.bb
create mode 100644 meta/recipes-connectivity/openssl/openssl_1.0.2i.bb
diff --git a/meta/recipes-connectivity/openssl/openssl.inc
b/meta/recipes-connectivity/openssl/openssl.inc
index a632d8a..f3a2c5a 100644
--- a/meta/recipes-connectivity/openssl/openssl.inc
+++ b/meta/recipes-connectivity/openssl/openssl.inc
@@ -211,6 +211,7 @@ do_install_ptest () {
ln -sf ${libdir}/ssl/misc/CA.sh ${D}${PTEST_PATH}/apps
ln -sf ${sysconfdir}/ssl/openssl.cnf ${D}${PTEST_PATH}/apps
ln -sf ${bindir}/openssl ${D}${PTEST_PATH}/apps
+ cp apps/server.pem ${D}${PTEST_PATH}/apps
cp apps/server2.pem ${D}${PTEST_PATH}/apps
mkdir -p ${D}${PTEST_PATH}/util
install util/opensslwrap.sh${D}${PTEST_PATH}/util
diff --git a/meta/recipes-connectivity/openssl/openssl/CVE-2016-2177.patch
b/meta/recipes-connectivity/openssl/openssl/CVE-2016-2177.patch
deleted file mode 100644
index df36d5f..000
--- a/meta/recipes-connectivity/openssl/openssl/CVE-2016-2177.patch
+++ /dev/null
@@ -1,286 +0,0 @@
-From a004e72b95835136d3f1ea90517f706c24c03da7 Mon Sep 17 00:00:00 2001
-From: Matt Caswell <m...@openssl.org>
-Date: Thu, 5 May 2016 11:10:26 +0100
-Subject: [PATCH] Avoid some undefined pointer arithmetic
-
-A common idiom in the codebase is:
-
-if (p + len > limit)
-{
-return; /* Too long */
-}
-
-Where "p" points to some malloc'd data of SIZE bytes and
-limit == p + SIZE
-
-"len" here could be from some externally supplied data (e.g. from a TLS
-message).
-
-The rules of C pointer arithmetic are such that "p + len" is only well
-defined where len <= SIZE. Therefore the above idiom is actually
-undefined behaviour.
-
-For example this could cause problems if some malloc implementation
-provides an address for "p" such that "p + len" actually overflows for
-values of len that are too big and therefore p + len < limit!
-
-Issue reported by Guido Vranken.
-
-CVE-2016-2177
-
-Reviewed-by: Rich Salz <rs...@openssl.org>
-
-Upstream-Status: Backport
-CVE: CVE-2016-2177
-
-Signed-off-by: Armin Kuster <akus...@mvista.com>
-
-
- ssl/s3_srvr.c | 14 +++---
- ssl/ssl_sess.c | 2 +-
- ssl/t1_lib.c | 56 ++--
- 3 files changed, 38 insertions(+), 34 deletions(-)
-
-diff --git a/ssl/s3_srvr.c b/ssl/s3_srvr.c
-index ab28702..ab7f690 100644
a/ssl/s3_srvr.c
-+++ b/ssl/s3_srvr.c
-@@ -980,7 +980,7 @@ int ssl3_get_client_hello(SSL *s)
-
- session_length = *(p + SSL3_RANDOM_SIZE);
-
--if (p + SSL3_RANDOM_SIZE + session_length + 1 >= d + n) {
-+if (SSL3_RANDOM_SIZE + session_length + 1 >= (d + n) - p) {
- al = SSL_AD_DECODE_ERROR;
- SSLerr(SSL_F_SSL3_GET_CLIENT_HELLO, SSL_R_LENGTH_TOO_SHORT);
- goto f_err;
-@@ -998,7 +998,7 @@ int ssl3_get_client_hello(SSL *s)
- /* get the session-id */
- j = *(p++);
-
--if (p + j > d + n) {
-+if ((d + n)
Re: [OE-core] [PATCH 0/1] openssl: update to 1.0.2i (CVE-2016-6304 and more)
On Fri, 2016-09-23 at 15:11 +0300, Alexander Kanavin wrote: > On 09/23/2016 01:27 PM, Patrick Ohly wrote: > > > > There is one FAIL: > > > > ../util/shlib_wrap.sh ./dtlstest ../apps/server.pem ../apps/server.pem > > Starting Test 0 > > Failed to load server certificate > > Unable to create SSL_CTX pair > > make[2]: Leaving directory '/usr/lib/openssl/ptest/test' > > FAIL: test_dtls > > > > That's because server.pem wasn't installed. I'll fix that. > > > > However, ptest-runner returns with 0, i.e. success? Should it do that? > > What does the failing test itself return? After checking the > ptest-runner source code, it shouldn't return 0 if one of the tests it > runs fails with a non-zero exit. openssl's test/Makefile is the culprit: alltests: @(for i in $(all-tests); do \ ( $(MAKE) $$i && echo "PASS: $$i" ) || echo "FAIL: $$i"; \ done) If any test fails, it'll print FAIL, but won't cause make to fail and thus the error never results in a non-zero exit code anywhere. Here's a version which reports the problem via the return code: alltests: @(result=0; for i in $(all-tests); do \ if $(MAKE) $$i; then echo "PASS: $$i"; else echo "FAIL: $$i"; result=1; fi; \ done; exit $$result) OpenSSL seems to rely on output checking. Not sure whether a patch changing that would be accepted. How are ptests used in the autobuilders? Does the return code of ptest-runner matter, or is the output checked for ^PASS|SKIP|FAIL? Speaking of the autobuilders and openssl-ptest in general, has no-one noticed before that occasionally tests fail because file time stamps imply that recompilation is needed? I got that a few times now and will send a fix. I'm just wondering why that wasn't a problem earlier. -- Best Regards, Patrick Ohly The content of this message is my personal opinion only and although I am an employee of Intel, the statements I make here in no way represent Intel's position on the issue, nor am I authorized to speak on behalf of Intel on this matter. -- ___ Openembedded-core mailing list Openembedded-core@lists.openembedded.org http://lists.openembedded.org/mailman/listinfo/openembedded-core
Re: [OE-core] [PATCH 0/1] openssl: update to 1.0.2i (CVE-2016-6304 and more)
On Fri, 2016-09-23 at 10:38 +0200, Patrick Ohly wrote: > Fixes several CVEs. > > It compiled for me okay for qemux86, but running the ptests showed a > problem in one of the new tests. I'll investigate that further There is one FAIL: ../util/shlib_wrap.sh ./dtlstest ../apps/server.pem ../apps/server.pem Starting Test 0 Failed to load server certificate Unable to create SSL_CTX pair make[2]: Leaving directory '/usr/lib/openssl/ptest/test' FAIL: test_dtls That's because server.pem wasn't installed. I'll fix that. However, ptest-runner returns with 0, i.e. success? Should it do that? -- Best Regards, Patrick Ohly The content of this message is my personal opinion only and although I am an employee of Intel, the statements I make here in no way represent Intel's position on the issue, nor am I authorized to speak on behalf of Intel on this matter. -- ___ Openembedded-core mailing list Openembedded-core@lists.openembedded.org http://lists.openembedded.org/mailman/listinfo/openembedded-core
[OE-core] [PATCH 1/1] openssl: update to 1.0.2i (CVE-2016-6304 and more)
This update fixes several CVEs:
* OCSP Status Request extension unbounded memory growth (CVE-2016-6304)
* SWEET32 Mitigation (CVE-2016-2183)
* OOB write in MDC2_Update() (CVE-2016-6303)
* Malformed SHA512 ticket DoS (CVE-2016-6302)
* OOB write in BN_bn2dec() (CVE-2016-2182)
* OOB read in TS_OBJ_print_bio() (CVE-2016-2180)
* DTLS buffered message DoS (CVE-2016-2179)
* DTLS replay protection DoS (CVE-2016-2181)
* Certificate message OOB reads (CVE-2016-6306)
Of these, only CVE-2016-6304 is considered of high
severity. Everything else is low. CVE-2016-2177 and CVE-2016-2178 were
already fixed via local patches, which can be removed now.
See https://www.openssl.org/news/secadv/20160922.txt for details.
Some patches had to be refreshed and one compile error fix from
upstream's OpenSSL_1_0_2-stable was required.
Signed-off-by: Patrick Ohly <patrick.o...@intel.com>
---
.../openssl/openssl/CVE-2016-2177.patch| 286 -
.../openssl/openssl/CVE-2016-2178.patch| 51
.../openssl/Fix-typo-introduced-by-a03f81f4.patch | 29 +++
.../openssl/openssl/debian/ca.patch| 2 +-
.../openssl/openssl/parallel.patch | 17 +-
.../{openssl_1.0.2h.bb => openssl_1.0.2i.bb} | 7 +-
6 files changed, 47 insertions(+), 345 deletions(-)
delete mode 100644
meta/recipes-connectivity/openssl/openssl/CVE-2016-2177.patch
delete mode 100644
meta/recipes-connectivity/openssl/openssl/CVE-2016-2178.patch
create mode 100644
meta/recipes-connectivity/openssl/openssl/Fix-typo-introduced-by-a03f81f4.patch
rename meta/recipes-connectivity/openssl/{openssl_1.0.2h.bb =>
openssl_1.0.2i.bb} (91%)
diff --git a/meta/recipes-connectivity/openssl/openssl/CVE-2016-2177.patch
b/meta/recipes-connectivity/openssl/openssl/CVE-2016-2177.patch
deleted file mode 100644
index df36d5f..000
--- a/meta/recipes-connectivity/openssl/openssl/CVE-2016-2177.patch
+++ /dev/null
@@ -1,286 +0,0 @@
-From a004e72b95835136d3f1ea90517f706c24c03da7 Mon Sep 17 00:00:00 2001
-From: Matt Caswell <m...@openssl.org>
-Date: Thu, 5 May 2016 11:10:26 +0100
-Subject: [PATCH] Avoid some undefined pointer arithmetic
-
-A common idiom in the codebase is:
-
-if (p + len > limit)
-{
-return; /* Too long */
-}
-
-Where "p" points to some malloc'd data of SIZE bytes and
-limit == p + SIZE
-
-"len" here could be from some externally supplied data (e.g. from a TLS
-message).
-
-The rules of C pointer arithmetic are such that "p + len" is only well
-defined where len <= SIZE. Therefore the above idiom is actually
-undefined behaviour.
-
-For example this could cause problems if some malloc implementation
-provides an address for "p" such that "p + len" actually overflows for
-values of len that are too big and therefore p + len < limit!
-
-Issue reported by Guido Vranken.
-
-CVE-2016-2177
-
-Reviewed-by: Rich Salz <rs...@openssl.org>
-
-Upstream-Status: Backport
-CVE: CVE-2016-2177
-
-Signed-off-by: Armin Kuster <akus...@mvista.com>
-
-
- ssl/s3_srvr.c | 14 +++---
- ssl/ssl_sess.c | 2 +-
- ssl/t1_lib.c | 56 ++--
- 3 files changed, 38 insertions(+), 34 deletions(-)
-
-diff --git a/ssl/s3_srvr.c b/ssl/s3_srvr.c
-index ab28702..ab7f690 100644
a/ssl/s3_srvr.c
-+++ b/ssl/s3_srvr.c
-@@ -980,7 +980,7 @@ int ssl3_get_client_hello(SSL *s)
-
- session_length = *(p + SSL3_RANDOM_SIZE);
-
--if (p + SSL3_RANDOM_SIZE + session_length + 1 >= d + n) {
-+if (SSL3_RANDOM_SIZE + session_length + 1 >= (d + n) - p) {
- al = SSL_AD_DECODE_ERROR;
- SSLerr(SSL_F_SSL3_GET_CLIENT_HELLO, SSL_R_LENGTH_TOO_SHORT);
- goto f_err;
-@@ -998,7 +998,7 @@ int ssl3_get_client_hello(SSL *s)
- /* get the session-id */
- j = *(p++);
-
--if (p + j > d + n) {
-+if ((d + n) - p < j) {
- al = SSL_AD_DECODE_ERROR;
- SSLerr(SSL_F_SSL3_GET_CLIENT_HELLO, SSL_R_LENGTH_TOO_SHORT);
- goto f_err;
-@@ -1054,14 +1054,14 @@ int ssl3_get_client_hello(SSL *s)
-
- if (SSL_IS_DTLS(s)) {
- /* cookie stuff */
--if (p + 1 > d + n) {
-+if ((d + n) - p < 1) {
- al = SSL_AD_DECODE_ERROR;
- SSLerr(SSL_F_SSL3_GET_CLIENT_HELLO, SSL_R_LENGTH_TOO_SHORT);
- goto f_err;
- }
- cookie_len = *(p++);
-
--if (p + cookie_len > d + n) {
-+if ((d + n ) - p < cookie_len) {
- al = SSL_AD_DECODE_ERROR;
- SSLerr(SSL_F_SSL3_GET_CLIENT_HELLO, SSL_R_LENGTH_TOO_SHORT);
- goto f_err;
-@@ -1131,7 +1131,7 @@ int ssl3_get_client_hello(SSL *s)
- }
- }
-
--if (p + 2 > d + n) {
-+if ((d + n ) - p < 2) {
- al = SSL_AD_DECODE_ERROR;
- SSLerr(SSL_F_SSL3_GET_CLIENT_HELLO, SSL_R_LENGTH_TOO_SHORT);
- goto f_err;
-@@ -
[OE-core] [PATCH 0/1] openssl: update to 1.0.2i (CVE-2016-6304 and more)
Fixes several CVEs.
It compiled for me okay for qemux86, but running the ptests showed a
problem in one of the new tests. I'll investigate that further, but in
the meantime wanted to send out the patch already in case that someone
has any comments, and to let everyone know that something is in the
pipeline for these CVEs.
The following changes since commit 7e0f95bf359bc3b5bb1578024a993e184de155cd:
base.bbclass: Drop unnecessary dirs setting (2016-09-22 11:08:23 +0100)
are available in the git repository at:
git://github.com/pohly/openembedded-core openssl-102i
https://github.com/pohly/openembedded-core/tree/openssl-102i
Patrick Ohly (1):
openssl: update to 1.0.2i (CVE-2016-6304 and more)
.../openssl/openssl/CVE-2016-2177.patch| 286 -
.../openssl/openssl/CVE-2016-2178.patch| 51
.../openssl/Fix-typo-introduced-by-a03f81f4.patch | 29 +++
.../openssl/openssl/debian/ca.patch| 2 +-
.../openssl/openssl/parallel.patch | 17 +-
.../{openssl_1.0.2h.bb => openssl_1.0.2i.bb} | 7 +-
6 files changed, 47 insertions(+), 345 deletions(-)
delete mode 100644
meta/recipes-connectivity/openssl/openssl/CVE-2016-2177.patch
delete mode 100644
meta/recipes-connectivity/openssl/openssl/CVE-2016-2178.patch
create mode 100644
meta/recipes-connectivity/openssl/openssl/Fix-typo-introduced-by-a03f81f4.patch
rename meta/recipes-connectivity/openssl/{openssl_1.0.2h.bb =>
openssl_1.0.2i.bb} (91%)
--
2.1.4
--
___
Openembedded-core mailing list
Openembedded-core@lists.openembedded.org
http://lists.openembedded.org/mailman/listinfo/openembedded-core
Re: [OE-core] rm_work + pybootchart enhancements
On Wed, 2016-09-21 at 17:39 +0200, Patrick Ohly wrote: > new-rmwork-new-scheduler > elapsed: 42:58.54 > final disk usage: 12873MiB > max disk usage: 14230MiB > > A bit better in terms of max disk usage than with the completion > scheduler, but not by much. My observation is that in practice there > aren't that many ready tasks, so often the choice of priorities among > them doesn't have such a big influence. The lower bound of the build > time is determined by the length of the do_configure->do_compile > critical path and it does not matter much when the other tasks run, as > long as they run at some point in parallel to that. Perhaps that is > different for larger distros. I might redo this benchmark with Ostro OS > XT... When building the ostro-xt-image-noswupd in the different configurations, I get these numbers: elapsed: 1:52:59 final disk usage: 135963MiB max disk usage: 136521MiB original-rmwork elapsed: 1:55:55 final disk usage: 33753MiB max disk usage: 66832MiB new-rmwork elapsed: 1:53:33 final disk usage: 33755MiB max disk usage: 69757MiB new-rmwork-new-scheduler elapsed: 1:53:13 final disk usage: 33754MiB max disk usage: 65933MiB new-rmwork-new-scheduler-additional-tasks elapsed: 1:53:57 final disk usage: 33754MiB max disk usage: 69173MiB Here the "rmwork" scheduler again shows some advantages over the "completion" scheduler (faster build, lower disk usage) as long as the "additional tasks" are turned off. Looks like doing additional work (like do_configure) is hurting more than it helps. -- Best Regards, Patrick Ohly The content of this message is my personal opinion only and although I am an employee of Intel, the statements I make here in no way represent Intel's position on the issue, nor am I authorized to speak on behalf of Intel on this matter. -- ___ Openembedded-core mailing list Openembedded-core@lists.openembedded.org http://lists.openembedded.org/mailman/listinfo/openembedded-core
Re: [OE-core] rm_work + pybootchart enhancements
On Thu, 2016-09-22 at 10:37 -0700, Christopher Larson wrote: > > On Thu, Sep 22, 2016 at 5:52 AM, Patrick Ohly <patrick.o...@intel.com> > wrote: > That's bitbake specific. What I was remembering is something > generic > that works for anything that prints test results on stdout. ... > I use https://github.com/rustyrussell/stats for that, quite handy. That's it, thanks. I even still had it in my $HOME/bin... now I just need remember it the next time that I need it. -- Best Regards, Patrick Ohly The content of this message is my personal opinion only and although I am an employee of Intel, the statements I make here in no way represent Intel's position on the issue, nor am I authorized to speak on behalf of Intel on this matter. -- ___ Openembedded-core mailing list Openembedded-core@lists.openembedded.org http://lists.openembedded.org/mailman/listinfo/openembedded-core
Re: [OE-core] rm_work + pybootchart enhancements
On Wed, 2016-09-21 at 18:08 +0200, Martin Jansa wrote: > On Wed, Sep 21, 2016 at 05:39:02PM +0200, Patrick Ohly wrote: > > Test script attached. I only ran this once, so I can't vouch that the > > numbers are stable. I remember reading about a benchmark wrapper which > > can run such a test script multiple times and then will automatically > > merge the output of all runs together, adding min/max/average/deviation, > > but couldn't find the tool again. Does anyone know which tool does that? > > it's in oe-core: > ./scripts/contrib/bb-perf/bb-matrix-plot.sh > ./scripts/contrib/bb-perf/bb-matrix.sh > > ... That's bitbake specific. What I was remembering is something generic that works for anything that prints test results on stdout. Perhaps I was dreaming ;-} -- Best Regards, Patrick Ohly The content of this message is my personal opinion only and although I am an employee of Intel, the statements I make here in no way represent Intel's position on the issue, nor am I authorized to speak on behalf of Intel on this matter. -- ___ Openembedded-core mailing list Openembedded-core@lists.openembedded.org http://lists.openembedded.org/mailman/listinfo/openembedded-core
Re: [OE-core] [oe-core][PATCH 1/1] systemd-compat-units: do not inherit allarch
On Mon, 2016-08-22 at 15:12 -0700, Joe Slater wrote:
> Even though we are just a script, we do depend on
> systemd being on the target and need an RDEPENDS
> which means we cannot also be allarch.
>
> Signed-off-by: Joe Slater <jsla...@windriver.com>
> ---
> meta/recipes-core/systemd/systemd-compat-units.bb |2 +-
> 1 file changed, 1 insertion(+), 1 deletion(-)
>
> diff --git a/meta/recipes-core/systemd/systemd-compat-units.bb
> b/meta/recipes-core/systemd/systemd-compat-units.bb
> index f246e83..421fc06 100644
> --- a/meta/recipes-core/systemd/systemd-compat-units.bb
> +++ b/meta/recipes-core/systemd/systemd-compat-units.bb
> @@ -9,7 +9,7 @@ DEPENDS = "systemd-systemctl-native"
>
> S = "${WORKDIR}"
>
> -inherit allarch distro_features_check
> +inherit distro_features_check
+1 from me. The alternative solution would be to add
"systemd-compat-units->systemd" to SIGGEN_EXCLUDE_SAFE_RECIPE_DEPS in
meta/conf/layer.conf, but that's more complicated and probably not worth
it.
--
Best Regards, Patrick Ohly
The content of this message is my personal opinion only and although
I am an employee of Intel, the statements I make here in no way
represent Intel's position on the issue, nor am I authorized to speak
on behalf of Intel on this matter.
--
___
Openembedded-core mailing list
Openembedded-core@lists.openembedded.org
http://lists.openembedded.org/mailman/listinfo/openembedded-core
[OE-core] [PATCH 0/1] static IDs bugs and usability
In Ostro OS we just started to use useradd-staticids.bbclass. Dynamic IDs are an error (USERADD_ERROR_DYNAMIC = "error") by default because warnings probably wouldn't be noticed. I was a bit reluctant to make this change because Ostro OS is supposed to be customizable, and that becomes harder when developers also have to define IDs. But we didn't have a choice because after updating to recent OE-core, our dynamically assigned IDs started to change randomly between builds, which (as we learned only because of that change) broke the swupd update mechanism. Personally I suspect that Python3's randomization feature is responsible for not getting deterministic results anymore, but I haven't investigated further. useradd-staticids.bbclass has a bug which impacts developers when they edit the table files to fix a warning or error raised by the class: the cached version of the affected recipe continues to be used and thus editing the files has no effect. Patch fixing that follows. There's also a caveat associated with how USERADD_ERROR_DYNAMIC = "error" is implemented: it marks recipes as "skipped" with an error messages from useradd-staticids.bbclass. This is not particularly developer-friendly. For example, when the skipped recipe is deep down in the dependency chain, then building will fail with rather difficult to understand "nothing provides..." messages. It does mention eventually that a recipe was skipped with the right error, so the information is at least there. However, it's also possible that a recipe exists in two versions, and only the one normally picked as default gets skipped. In that case, the remaining version will get build, i.e. useradd-staticids.bbclass has the very undesirable effect of silently building something else than expected instead of aborting the build because of the error. I've not seen that in practice, though, so I don't know how big of a problem it is. The following changes since commit b32d430c3c7dccf3a8d06ab492d648893a05950f: dpkg: use snapshot.debian.org for SRC_URI (2016-07-26 08:56:08 +0100) are available in the git repository at: git://github.com/pohly/openembedded-core static-ids https://github.com/pohly/openembedded-core/tree/static-ids Patrick Ohly (1): useradd-staticids.bbclass: trigger reparsing when table files change meta/classes/useradd-staticids.bbclass | 13 + 1 file changed, 13 insertions(+) -- 2.1.4 -- ___ Openembedded-core mailing list Openembedded-core@lists.openembedded.org http://lists.openembedded.org/mailman/listinfo/openembedded-core
[OE-core] [PATCH 1/1] useradd-staticids.bbclass: trigger reparsing when table files change
This addresses (among others) the following problem:
- USERADD_ERROR_DYNAMIC=error causes a recipe to get skipped
because a static ID entry is missing
- the entry gets added to the file
- using the recipe still fails with the same error as before
because the recipe gets loaded from the cache instead
of re-parsing it with the new table content
Signed-off-by: Patrick Ohly <patrick.o...@intel.com>
---
meta/classes/useradd-staticids.bbclass | 13 +
1 file changed, 13 insertions(+)
diff --git a/meta/classes/useradd-staticids.bbclass
b/meta/classes/useradd-staticids.bbclass
index 149245b..46d4a4b 100644
--- a/meta/classes/useradd-staticids.bbclass
+++ b/meta/classes/useradd-staticids.bbclass
@@ -285,6 +285,19 @@ def update_useradd_static_config(d):
return ";".join(newparams).strip()
+# The parsing of the current recipe depends on the content of
+# the files listed in USERADD_UID/GID_TABLES. We need to tell bitbake
+# about that explicitly to trigger re-parsing and thus re-execution of
+# this code when the files change.
+bbpath = d.getVar('BBPATH', True)
+for varname, default in (('USERADD_UID_TABLES', 'files/passwd'),
+ ('USERADD_GID_TABLES', 'files/group')):
+tables = d.getVar(varname, True)
+if not tables:
+tables = default
+for conf_file in tables.split():
+bb.parse.mark_dependency(d, bb.utils.which(bbpath, conf_file))
+
# Load and process the users and groups, rewriting the adduser/addgroup
params
useradd_packages = d.getVar('USERADD_PACKAGES', True)
--
2.1.4
--
___
Openembedded-core mailing list
Openembedded-core@lists.openembedded.org
http://lists.openembedded.org/mailman/listinfo/openembedded-core
Re: [OE-core] [PATCH 1/2] meta-ostro-fixes: initramfs-framework: add retry loop for slow boot devices (like USB)
On Mon, 2016-07-11 at 12:17 -0300, Otavio Salvador wrote: > On Mon, Jul 11, 2016 at 12:10 PM, Patrick Ohly <patrick.o...@intel.com> wrote: > > On Mon, 2016-07-11 at 11:35 -0300, Otavio Salvador wrote: > >> Hello Ross, > >> > >> On Fri, Jul 1, 2016 at 2:50 PM, Otavio Salvador <ota...@ossystems.com.br> > >> wrote: > >> > Agreed but please split the rootfs in another module; so we don't > >> > force it to be included. The e2fs can rdepend on it. > >> > >> I noticed this has been merged but my comment here was totally ignored. > >> Why? > > > > I can't speak for Ross, but perhaps he felt that it was already a > > worthwhile improvement. I wanted to address your comment (and still do), > > but didn't find the time last week. I'll do it via another patch now, > > okay? > > Ok but Ross action is wrong in my opinion, in worse case he should > have asked before. If comments are ignored there is no point in people > spend time doing so. > > I wanted the change to be atomic and not two changes which may > required more rework for same logic change. :-( It turned out to be just a mistake that it was already merge. On the other hand, one can also argue that these are two separate logical changes - adding the retry loop, and then making that code just the optional default implementation for mounting. Anyway, see "initramfs-framework: make rootfs module optional" for that second part. Note that I did not make e2fs depend on rootfs, because there's no real connection between the two (for example, the rootfs could also be btrfs). Instead it is the finish script which depends on some kind of rootfs mounting. -- Best Regards, Patrick Ohly The content of this message is my personal opinion only and although I am an employee of Intel, the statements I make here in no way represent Intel's position on the issue, nor am I authorized to speak on behalf of Intel on this matter. -- ___ Openembedded-core mailing list Openembedded-core@lists.openembedded.org http://lists.openembedded.org/mailman/listinfo/openembedded-core
[OE-core] [PATCH] initramfs-framework: make rootfs module optional
It still gets installed by default via RRECOMMENDS without having to update
users of the framework (because without it, the framework is incomplete),
but that recommendation can be overridden on a per-image basis.
Signed-off-by: Patrick Ohly <patrick.o...@intel.com>
---
.../recipes-core/initrdscripts/initramfs-framework_1.0.bb | 15 ++-
1 file changed, 14 insertions(+), 1 deletion(-)
diff --git a/meta/recipes-core/initrdscripts/initramfs-framework_1.0.bb
b/meta/recipes-core/initrdscripts/initramfs-framework_1.0.bb
index 89e153d..67a1b04 100644
--- a/meta/recipes-core/initrdscripts/initramfs-framework_1.0.bb
+++ b/meta/recipes-core/initrdscripts/initramfs-framework_1.0.bb
@@ -47,9 +47,18 @@ PACKAGES = "${PN}-base \
initramfs-module-mdev \
initramfs-module-udev \
initramfs-module-e2fs \
+initramfs-module-rootfs \
initramfs-module-debug"
-FILES_${PN}-base = "/init /init.d/90-rootfs /init.d/99-finish /dev"
+FILES_${PN}-base = "/init /init.d/99-finish /dev"
+
+# 99-finish in base depends on some other module which mounts
+# the rootfs, like 90-rootfs. To replace that default, use
+# BAD_RECOMMENDATIONS += "initramfs-module-rootfs" in your
+# initramfs recipe and install something else, or install
+# something that runs earlier (for example, a 89-my-rootfs)
+# and mounts the rootfs. Then 90-rootfs will proceed immediately.
+RRECOMMENDS_${PN}-base += "initramfs-module-rootfs"
SUMMARY_initramfs-module-mdev = "initramfs support for mdev"
RDEPENDS_initramfs-module-mdev = "${PN}-base busybox-mdev"
@@ -63,6 +72,10 @@ SUMMARY_initramfs-module-e2fs = "initramfs support for
ext4/ext3/ext2 filesystem
RDEPENDS_initramfs-module-e2fs = "${PN}-base"
FILES_initramfs-module-e2fs = "/init.d/10-e2fs"
+SUMMARY_initramfs-module-rootfs = "initramfs support for locating and mounting
the root partition"
+RDEPENDS_initramfs-module-rootfs = "${PN}-base"
+FILES_initramfs-module-rootfs = "/init.d/90-rootfs"
+
SUMMARY_initramfs-module-debug = "initramfs dynamic debug support"
RDEPENDS_initramfs-module-debug = "${PN}-base"
FILES_initramfs-module-debug = "/init.d/00-debug"
--
2.6.2
--
___
Openembedded-core mailing list
Openembedded-core@lists.openembedded.org
http://lists.openembedded.org/mailman/listinfo/openembedded-core
Re: [OE-core] [PATCH 1/2] meta-ostro-fixes: initramfs-framework: add retry loop for slow boot devices (like USB)
On Mon, 2016-07-11 at 11:35 -0300, Otavio Salvador wrote: > Hello Ross, > > On Fri, Jul 1, 2016 at 2:50 PM, Otavio Salvador <ota...@ossystems.com.br> > wrote: > > Agreed but please split the rootfs in another module; so we don't > > force it to be included. The e2fs can rdepend on it. > > I noticed this has been merged but my comment here was totally ignored. Why? I can't speak for Ross, but perhaps he felt that it was already a worthwhile improvement. I wanted to address your comment (and still do), but didn't find the time last week. I'll do it via another patch now, okay? -- Best Regards, Patrick Ohly The content of this message is my personal opinion only and although I am an employee of Intel, the statements I make here in no way represent Intel's position on the issue, nor am I authorized to speak on behalf of Intel on this matter. -- ___ Openembedded-core mailing list Openembedded-core@lists.openembedded.org http://lists.openembedded.org/mailman/listinfo/openembedded-core
[OE-core] [PATCH 2/2] meta-ostro-fixes: initramfs-framework: Add support for PartUUIDs
From: Igor Stoppa <igor.sto...@intel.com>
The rootfs can be addressed also by referring to the PartUUID
value from the GPT.
This patch enables such type of reference.
Signed-off-by: Igor Stoppa <igor.sto...@intel.com>
Signed-off-by: Patrick Ohly <patrick.o...@intel.com>
---
meta/recipes-core/initrdscripts/initramfs-framework/rootfs | 5 +
1 file changed, 5 insertions(+)
diff --git a/meta/recipes-core/initrdscripts/initramfs-framework/rootfs
b/meta/recipes-core/initrdscripts/initramfs-framework/rootfs
index 5790d8c..14768f1 100644
--- a/meta/recipes-core/initrdscripts/initramfs-framework/rootfs
+++ b/meta/recipes-core/initrdscripts/initramfs-framework/rootfs
@@ -26,6 +26,11 @@ rootfs_run() {
bootparam_root="/dev/disk/by-uuid/$root_uuid"
fi
+ if [ "`echo ${bootparam_root} | cut -c1-9`" =
"PARTUUID=" ]; then
+ root_uuid=`echo $bootparam_root | cut -c10-`
+
bootparam_root="/dev/disk/by-partuuid/$root_uuid"
+ fi
+
if [ -e "$bootparam_root" ]; then
flags=""
if [ -n "$bootparam_ro" ] && ! echo
"$bootparam_rootflags" | grep -w -q "ro"; then
--
2.1.4
--
___
Openembedded-core mailing list
Openembedded-core@lists.openembedded.org
http://lists.openembedded.org/mailman/listinfo/openembedded-core
[OE-core] [PATCH 1/2] meta-ostro-fixes: initramfs-framework: add retry loop for slow boot devices (like USB)
On some hardware platforms (Gigabyte, qemu), detection of USB devices
by the kernel is slow enough such that it happens only after the first
attempt to mount the rootfs. We need to keep trying for a while
(default: 5s seconds, controlled by roottimeout=) and sleep
between each attempt (default: one second, rootdelay=).
This change intentionally splits finding the rootfs (in the new
"rootfs") and switching to it ("finish"). That is needed to keep udev
running while waiting for the rootfs, because it shuts down before
"finish" starts. It is also the direction that was discussed on the OE
mailing list for future changes to initramfs-framework (like
supporting a "live CD" module, which would replace or further augment
mounting of the rootfs).
Signed-off-by: Patrick Ohly <patrick.o...@intel.com>
---
.../initrdscripts/initramfs-framework/finish | 33 -
.../initrdscripts/initramfs-framework/rootfs | 57 ++
.../initrdscripts/initramfs-framework_1.0.bb | 4 +-
3 files changed, 60 insertions(+), 34 deletions(-)
create mode 100644 meta/recipes-core/initrdscripts/initramfs-framework/rootfs
diff --git a/meta/recipes-core/initrdscripts/initramfs-framework/finish
b/meta/recipes-core/initrdscripts/initramfs-framework/finish
index d09bbb8..717383e 100755
--- a/meta/recipes-core/initrdscripts/initramfs-framework/finish
+++ b/meta/recipes-core/initrdscripts/initramfs-framework/finish
@@ -8,39 +8,6 @@ finish_enabled() {
finish_run() {
if [ -n "$ROOTFS_DIR" ]; then
- if [ -n "$bootparam_rootdelay" ]; then
- debug "Sleeping for $rootdelay second(s) to wait root
to settle..."
- sleep $bootparam_rootdelay
- fi
-
- if [ -n "$bootparam_root" ]; then
- debug "No e2fs compatible filesystem has been mounted,
mounting $bootparam_root..."
-
- if [ "`echo ${bootparam_root} | cut -c1-5`" = "UUID="
]; then
- root_uuid=`echo $bootparam_root | cut -c6-`
- bootparam_root="/dev/disk/by-uuid/$root_uuid"
- fi
-
- if [ -e "$bootparam_root" ]; then
- flags=""
- if [ -n "$bootparam_ro" ]; then
- if [ -n "$bootparam_rootflags" ]; then
-
bootparam_rootflags="$bootparam_rootflags,"
- fi
-
bootparam_rootflags="${bootparam_rootflags}ro"
- fi
- if [ -n "$bootparam_rootflags" ]; then
- flags="$flags -o$bootparam_rootflags"
- fi
- if [ -n "$bootparam_rootfstype" ]; then
- flags="$flags -t$bootparam_rootfstype"
- fi
- mount $flags $bootparam_root $ROOTFS_DIR
- else
- msg "root '$bootparam_root' doesn't exist."
- fi
- fi
-
if [ ! -d $ROOTFS_DIR/dev ]; then
fatal "ERROR: There's no '/dev' on rootfs."
fi
diff --git a/meta/recipes-core/initrdscripts/initramfs-framework/rootfs
b/meta/recipes-core/initrdscripts/initramfs-framework/rootfs
new file mode 100644
index 000..5790d8c
--- /dev/null
+++ b/meta/recipes-core/initrdscripts/initramfs-framework/rootfs
@@ -0,0 +1,57 @@
+#!/bin/sh
+# Copyright (C) 2011 O.S. Systems Software LTDA.
+# Licensed on MIT
+
+rootfs_enabled() {
+ return 0
+}
+
+rootfs_run() {
+if [ -z "$ROOTFS_DIR" ]; then
+ return
+fi
+ C=0
+ delay=${bootparam_rootdelay:-1}
+ timeout=${bootparam_roottimeout:-5}
+ while [ ! -d $ROOTFS_DIR/dev ]; do
+ if [ $(( $C * $delay )) -gt $timeout ]; then
+ fatal "root '$bootparam_root' doesn't exist or does not
contain a /dev."
+ fi
+
+ if [ -n "$bootparam_root" ]; then
+ debug "No e2fs compatible filesystem has been mounted,
mounting $bootparam_root..."
+
+ if [ "`echo ${bootparam_root} | cut -c1-5`" = "UUID="
]; then
+ root_uuid=`echo $bootparam_root | cut -c6-`
+ bootparam_root="/dev/disk/by-uuid/$root_uuid"
[OE-core] [PATCH 0/2] initramfs-framework enhancements
These are two enhancements that we've used for a while now in Ostro OS: - a retry loop that we found necessary for some slow USB hardware - finding the root partition by partition UUID (only filesystem UUID supported before) The following changes since commit 646c366c2566bd8dd6f73681cea9f5b021589a56: gst-player: upgrade to latest HEAD (2016-06-27 14:08:37 +0100) are available in the git repository at: git://github.com/pohly/openembedded-core initramfs-framework https://github.com/pohly/openembedded-core/tree/initramfs-framework Igor Stoppa (1): meta-ostro-fixes: initramfs-framework: Add support for PartUUIDs Patrick Ohly (1): meta-ostro-fixes: initramfs-framework: add retry loop for slow boot devices (like USB) .../initrdscripts/initramfs-framework/finish | 33 .../initrdscripts/initramfs-framework/rootfs | 62 ++ .../initrdscripts/initramfs-framework_1.0.bb | 4 +- 3 files changed, 65 insertions(+), 34 deletions(-) create mode 100644 meta/recipes-core/initrdscripts/initramfs-framework/rootfs -- 2.1.4 -- ___ Openembedded-core mailing list Openembedded-core@lists.openembedded.org http://lists.openembedded.org/mailman/listinfo/openembedded-core
Re: [OE-core] [PATCH 1/2] rootfs-postcommands: don't hard-code .rootfs in manifest filename
On Mon, 2016-06-27 at 08:43 +0100, Joshua G Lock wrote: > On Wed, 2016-06-22 at 11:22 +0100, Joshua Lock wrote: > > The .rootfs suffix of a generated image can be overridden by setting > > IMAGE_NAME_SUFFIX, to ensure manifest filenames match the associated > > image make use of this variable in the manifest filename. > > This change will break a few external layers so probably isn't worth it > for a minor cosmetic change? Agreed, please don't merge yet. The problem is a bit more complicated: is IMAGE_MANIFEST the manifest file for the entire image (as the name implies), or just of the main partition? The two are not necessarily the same, because some image types combine the main partition with additional content (like a boot loader and the kernel). This conceptual issue needs further work. -- Best Regards, Patrick Ohly The content of this message is my personal opinion only and although I am an employee of Intel, the statements I make here in no way represent Intel's position on the issue, nor am I authorized to speak on behalf of Intel on this matter. -- ___ Openembedded-core mailing list Openembedded-core@lists.openembedded.org http://lists.openembedded.org/mailman/listinfo/openembedded-core
Re: [OE-core] [PATCH] classes/rm_work: use the idle I/O scheduler class
On Tue, 2016-06-14 at 16:18 +0100, Ross Burton wrote: > As rm_work is just cleanup it shouldn't starve more important tasks such as > do_compile of I/O, so use BB_TASK_IONICE_LEVEL to run the task in the idle > scheduler class. Whether that's desirable depends a lot on the goals for rm_work: when I tried to use it for TravisCI to get around some pretty tight disk space constraints, I found that do_rm_work was often not scheduled early enough because other tasks generating more files had higher priority. Reducing the IO priority of do_rm_work may have the same effect: it runs, but then instead of removing files, the system produces more of them, thus increasing the risk of exhausting the disk space. I suspect a lot of benchmarking will be needed to determine what really works well and what doesn't. I ended up writing a custom scheduler for running under TravisCI: https://github.com/01org/meta-intel-iot-security/blob/master/scripts/rmwork.py That orders do_rm_work before any other task and also orders all tasks related to single recipe so that they run together, thus making it possible to clean up after do_build sooner. As an additional tweak it distinguishes between "compile" and "cleanup" tasks and can run "cleanup" tasks when the normal scheduler wouldn't because BB_NUMBER_THREADS is reached. But it has the same problem: not enough benchmarking to really quantify the effect. All I know is that I stopped running out of disk space under TravisCI ;-} -- Best Regards, Patrick Ohly The content of this message is my personal opinion only and although I am an employee of Intel, the statements I make here in no way represent Intel's position on the issue, nor am I authorized to speak on behalf of Intel on this matter. -- ___ Openembedded-core mailing list Openembedded-core@lists.openembedded.org http://lists.openembedded.org/mailman/listinfo/openembedded-core
Re: [OE-core] State of bitbake world 2016-06-13
On Wed, 2016-06-15 at 12:45 +0200, Martin Jansa wrote: > libgphoto2, v4l-utils fail because recent upgrade of libjpeg-turbo in oe-core > (PNBLACKLIST sent) I have patches ready for that, will send them now. -- Best Regards, Patrick Ohly The content of this message is my personal opinion only and although I am an employee of Intel, the statements I make here in no way represent Intel's position on the issue, nor am I authorized to speak on behalf of Intel on this matter. -- ___ Openembedded-core mailing list Openembedded-core@lists.openembedded.org http://lists.openembedded.org/mailman/listinfo/openembedded-core
[OE-core] [PATCH] npm.bbclass: avoid str/byte conversion problems for PKGV and SUMMARY
In Python3, str.encode() returns byte strings, which later are not
converted back to strings automatically, leading to "TypeError: Can't
convert 'bytes' object to str implicitly" in code which reads PKGV and
SUMMARY and expects to find strings there.
The npm.bbclass must use values for d.setVar() that meet that
expectation, and thus the redundant (and in Python3, harmful)
.encode() gets removed.
Signed-off-by: Patrick Ohly <patrick.o...@intel.com>
---
meta/classes/npm.bbclass | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/meta/classes/npm.bbclass b/meta/classes/npm.bbclass
index d0d3d8f..95be751 100644
--- a/meta/classes/npm.bbclass
+++ b/meta/classes/npm.bbclass
@@ -46,10 +46,10 @@ python populate_packages_prepend () {
if pdata:
version = pdata.get('version', None)
if version:
-d.setVar('PKGV_%s' % expanded_pkgname, version.encode("utf8"))
+d.setVar('PKGV_%s' % expanded_pkgname, version)
description = pdata.get('description', None)
if description:
-d.setVar('SUMMARY_%s' % expanded_pkgname,
description.replace(u"\u2018", "'").replace(u"\u2019", "'").encode("utf8"))
+d.setVar('SUMMARY_%s' % expanded_pkgname,
description.replace(u"\u2018", "'").replace(u"\u2019", "'"))
d.appendVar('RDEPENDS_%s' % d.getVar('PN', True), ' %s' % '
'.join(pkgnames).replace('_', '-'))
}
--
2.1.4
--
___
Openembedded-core mailing list
Openembedded-core@lists.openembedded.org
http://lists.openembedded.org/mailman/listinfo/openembedded-core
Re: [OE-core] [PATCHv3] mkelfimage: obey LDFLAGS, sort out HOST_ flags
On Mon, 2016-05-23 at 04:52 +, Khem Raj wrote: > I am getting these errors with clang now > > > http://errors.yoctoproject.org/Errors/Details/64935/ And I am getting the same link error also during normal builds for Beaglebone (but not other platforms): https://ostroproject.org/jenkins/job/build_beaglebone/1663/console ... arm-ostro-linux-gnueabi-gcc -march=armv7-a -mfpu=neon -mfloat-abi=hard -mcpu=cortex-a8 --sysroot=/var/lib/jenkins/ostro-worker-17-slot-1-JopZL/ostro-os/build/tmp-glibc/sysroots/beaglebone -O2 -pipe -g -feliminate-unused-debug-types -fdebug-prefix-map=/var/lib/jenkins/ostro-worker-17-slot-1-JopZL/ostro-os/build/tmp-glibc/work/cortexa8hf-neon-ostro-linux-gnueabi/mkelfimage/4.0+gitAUTOINC+686a48a339-r0=/usr/src/debug/mkelfimage/4.0+gitAUTOINC+686a48a339-r0 -fdebug-prefix-map=/var/lib/jenkins/ostro-worker-17-slot-1-JopZL/ostro-os/build/tmp-glibc/sysroots/x86_64-linux= -fdebug-prefix-map=/var/lib/jenkins/ostro-worker-17-slot-1-JopZL/ostro-os/build/tmp-glibc/sysroots/beaglebone= -fstack-protector-strong -pie -fpie -D_FORTIFY_SOURCE=2 -Wformat -Wformat-security -Werror=format-security -fno-stack-protector -Wall -DVERSION='"2.7"' -DRELEASE_DATE='"27 March 2006"' -I include -DPACKAGE_NAME=\"\" -DPACKAGE_TARNAME=\"\" -DPACKAGE_VERSION=\"\" -DPACKAGE_STRING=\"\" -DPACKAGE_BUGREPORT=\ "\" -DPACKAGE_URL=\"\" -DSTDC_HEADERS=1 -DHAVE_SYS_TYPES_H=1 -DHAVE_SYS_STAT_H=1 -DHAVE_STDLIB_H=1 -DHAVE_STRING_H=1 -DHAVE_MEMORY_H=1 -DHAVE_STRINGS_H=1 -DHAVE_INTTYPES_H=1 -DHAVE_STDINT_H=1 -DHAVE_UNISTD_H=1 -DHAVE_ZLIB_H=1 -Wl,-O1 -Wl,--hash-style=gnu -Wl,--as-needed -fstack-protector-strong -Wl,-z,relro,-z,now objdir/main/mkelfImage.o objdir/linux-i386/mkelf-linux-i386.o objdir/linux-ia64/mkelf-linux-ia64.o -o objdir/sbin/mkelfImage -lz /var/lib/jenkins/ostro-worker-17-slot-1-JopZL/ostro-os/build/tmp-glibc/sysroots/x86_64-linux/usr/bin/arm-ostro-linux-gnueabi/../../libexec/arm-ostro-linux-gnueabi/gcc/arm-ostro-linux-gnueabi/5.3.0/ld: objdir/linux-i386/mkelf-linux-i386.o: relocation R_ARM_MOVW_ABS_NC against `a local symbol' can not be used when making a shared object; recompile with -fPIC objdir/linux-i386/mkelf-linux-i386.o: error adding symbols: Bad value collect2: error: ld returned 1 exit status main/Makefile:6: recipe for target 'objdir/sbin/mkelfImage' failed ... Reverting the commit fixes that, so can we just do that in OE-core master while the problem gets sorted out? -- Best Regards, Patrick Ohly The content of this message is my personal opinion only and although I am an employee of Intel, the statements I make here in no way represent Intel's position on the issue, nor am I authorized to speak on behalf of Intel on this matter. -- ___ Openembedded-core mailing list Openembedded-core@lists.openembedded.org http://lists.openembedded.org/mailman/listinfo/openembedded-core
Re: [OE-core] [PATCH 1/5] useradd.bbclass: Strip trailing '; ' in cmd params
On Wed, 2016-05-18 at 12:38 -0700, Khem Raj wrote: > This fix was not sufficient and there has to be extra fix done to also > consider the case > when sysroot is created during image build time. I have updated it in > pull request here > > http://git.openembedded.org/openembedded-core-contrib/commit/?h=kraj/pu=642c6cf0b6a0371de476513162bd0cefa9c438b3 That now fails for cases where the USERADD_PARAM ends in trailing white space after the last valid opts. The reason is that the 'if test "x $opts" = "x$remaining"; then break; fi' check does not get triggered anymore, because $remaining has the trailing white space whereas "$opts" does not. I'm not sure what the best approach is in this case. Perhaps also strip trailing whitespace from all assignments to $remaining? The case were we see this is: USERADD_PARAM_iot-rest-api-server="--system --home /var/lib/empty --no-create-home --shell /bin/false --gid restful restful " The build then hangs because the useradd preinst loops forever. -- Best Regards, Patrick Ohly The content of this message is my personal opinion only and although I am an employee of Intel, the statements I make here in no way represent Intel's position on the issue, nor am I authorized to speak on behalf of Intel on this matter. -- ___ Openembedded-core mailing list Openembedded-core@lists.openembedded.org http://lists.openembedded.org/mailman/listinfo/openembedded-core
[OE-core] [PATCH 2/2] combo-layer: avoid too long command lines in update with history
As suspected, invoking "git archive" with all intended files as
parameters can run into command line length limitations. Splitting up
the parameters into multiple invocations (xargs-style) works and was
tested after encountering the situation in practice.
Signed-off-by: Patrick Ohly <patrick.o...@intel.com>
---
scripts/combo-layer | 31 +--
1 file changed, 29 insertions(+), 2 deletions(-)
diff --git a/scripts/combo-layer b/scripts/combo-layer
index a0a737d..1ca2ce6 100755
--- a/scripts/combo-layer
+++ b/scripts/combo-layer
@@ -1266,8 +1266,35 @@ def apply_commit(parent, rev, largs, wargs, dest_dir,
file_filter=None):
target = os.path.join(wargs["destdir"], dest_dir)
if not os.path.isdir(target):
os.makedirs(target)
-runcmd("git archive %s %s | tar -C %s -xf -" % (rev, '
'.join([pipes.quote(x) for x in update]), pipes.quote(target)), **largs)
-runcmd("git add -f".split() + [os.path.join(dest_dir, x) for x in
update], **wargs)
+quoted_target = pipes.quote(target)
+# os.sysconf('SC_ARG_MAX') is lying: running a command with
+# string length 629343 already failed with "Argument list too
+# long" although SC_ARG_MAX = 2097152. "man execve" explains
+# the limitations, but those are pretty complicated. So here
+# we just hard-code a fixed value which is more likely to work.
+max_cmdsize = 64 * 1024
+while update:
+quoted_args = []
+unquoted_args = []
+cmdsize = 100 + len(quoted_target)
+while update:
+quoted_next = pipes.quote(update[0])
+size_next = len(quoted_next) + len(dest_dir) + 1
+logger.debug('cmdline length %d + %d < %d?' % (cmdsize,
size_next, os.sysconf('SC_ARG_MAX')))
+if cmdsize + size_next < max_cmdsize:
+quoted_args.append(quoted_next)
+unquoted_args.append(update.pop(0))
+cmdsize += size_next
+else:
+logger.debug('Breaking the cmdline at length %d' % cmdsize)
+break
+logger.debug('Final cmdline length %d / %d' % (cmdsize,
os.sysconf('SC_ARG_MAX')))
+cmd = "git archive %s %s | tar -C %s -xf -" % (rev, '
'.join(quoted_args), quoted_target)
+logger.debug('First cmdline length %d' % len(cmd))
+runcmd(cmd, **largs)
+cmd = "git add -f".split() + [os.path.join(dest_dir, x) for x in
unquoted_args]
+logger.debug('Second cmdline length %d' % reduce(lambda x, y: x +
len(y), cmd, 0))
+runcmd(cmd, **wargs)
if delete:
for path in delete:
if dest_dir:
--
2.1.4
--
___
Openembedded-core mailing list
Openembedded-core@lists.openembedded.org
http://lists.openembedded.org/mailman/listinfo/openembedded-core
[OE-core] [PATCH 0/2] fixes for combo-layer update history mode
The first fix is urgent: the default mode was accidentally changed from the traditional "apply patches" to "import the history". The second fix is less urgent but still useful for those who actually use the new mode. The following changes since commit fcaaabb401fffcda4db9a7d1f927a2a404e4776d: gcc-runtime, libgcc: Symlink c++ header and startup files in target_triplet for SDK use (2016-05-13 13:40:53 +0100) are available in the git repository at: git://github.com/pohly/openembedded-core combo-layer-history-fixes https://github.com/pohly/openembedded-core/tree/combo-layer-history-fixes Patrick Ohly (2): combo-layer: fix default "update" mode combo-layer: avoid too long command lines in update with history scripts/combo-layer | 34 ++ 1 file changed, 30 insertions(+), 4 deletions(-) -- 2.1.4 -- ___ Openembedded-core mailing list Openembedded-core@lists.openembedded.org http://lists.openembedded.org/mailman/listinfo/openembedded-core
[OE-core] [PATCH 1/2] combo-layer: fix default "update" mode
When the "history" option is not set in the combo-layer.conf, the
intended default was to use the traditional method. Passing "True" as
default when querying the config was unintentional.
Also remove some left-over debugging code.
Signed-off-by: Patrick Ohly <patrick.o...@intel.com>
---
scripts/combo-layer | 3 +--
1 file changed, 1 insertion(+), 2 deletions(-)
diff --git a/scripts/combo-layer b/scripts/combo-layer
index 92525ca..a0a737d 100755
--- a/scripts/combo-layer
+++ b/scripts/combo-layer
@@ -641,8 +641,7 @@ def action_update(conf, args):
history = None
for name in repos:
repo = conf.repos[name]
-repo_history = repo.get('history', True)
-logger.error('%s: %s' % (name, repo_history))
+repo_history = repo.get('history', False)
if history is None:
history = repo_history
elif history != repo_history:
--
2.1.4
--
___
Openembedded-core mailing list
Openembedded-core@lists.openembedded.org
http://lists.openembedded.org/mailman/listinfo/openembedded-core
Re: [OE-core] [PATCH] lib/oe/rootfs: Fix DEBUGFS generation when using opkg
On Mon, 2016-05-02 at 11:15 -0500, Alejandro del Castillo wrote: > > On 05/02/2016 08:39 AM, Patrick Ohly wrote: > > On Fri, 2016-04-22 at 14:51 +0100, Richard Purdie wrote: > >> When enabling extra DEBUGFS image generation with opkg, errors are seen > >> like: > >> > >> ERROR: core-image-minimal-1.0-r0 do_rootfs: Cannot get the installed > >> packages list. Command > >> '/media/build1/poky/build/tmp/sysroots/x86_64-linux/usr/bin/opkg -f > >> /media/build1/poky/build/tmp/work/qemux86_64-poky-linux/core-image-minimal/1.0-r0/opkg.conf > >> -o > >> /media/build1/poky/build/tmp/work/qemux86_64-poky-linux/core-image-minimal/1.0-r0/rootfs > >> --force_postinstall --prefer-arch-to-version status' returned 0 and > >> stderr: > >> Collected errors: > >> * file_md5sum_alloc: Failed to open > >> file > >> /media/build1/poky/build/tmp/work/qemux86_64-poky-linux/core-image-minimal/1.0-r0/rootfs/etc/syslog-startup.conf.busybox: > >> No such file or directory. > >> * file_md5sum_alloc: Failed to open > >> file > >> /media/build1/poky/build/tmp/work/qemux86_64-poky-linux/core-image-minimal/1.0-r0/rootfs/etc/fstab: > >> No such file or directory. > >> > >> basically for all CONFFILES in the image. This is due to the file > >> rearranging > >> the rootfs generation code does. If we preserve the /etc directory, > >> the avoids the problem. > > > > It avoids the problem, but it does not address the root cause (IMHO). > > Should opkg really complain about missing configuration files? It is > > perhaps an edge case, but besides editing a configuration file > > *removing* it entirely may also be a valid user modification. > > > > The usage of opkg here is during image building, but the operation > > itself (opkg status) is not specific to image creation and thus should > > be able to handle arbitrary states of /etc and the config files in > > general. > > Opkg status is supposed to give you the state of all your installed packages. > Currently is pretty rough data: it dumps into stdout the internal metadata > that > tracks package info and promotes to error any inconsistency, like conffiles > not > being present. But how does opkg know that "config file not present" is an error? It depends on the the semantic of the file and thus the package providing the file. It's not declared explicitly, so it is a fairly arbitrary judgment call to treat "missing" as error. > Looking at pacakge_manager.py, opkg status is being called on OpkgPkgsList. > Seems to me that if the purpose is to get all installed packages, then "opkg > list-installed" is the correct command. Looking at dpkg (DpkgPkgsList), > "dpkg-query -W" is being called, which I think maps to opkg list-installed, > not > to opkg status. That indeed seems better: output should be smaller, faster to run and it avoids the ambiguity about missing config files. -- Best Regards, Patrick Ohly The content of this message is my personal opinion only and although I am an employee of Intel, the statements I make here in no way represent Intel's position on the issue, nor am I authorized to speak on behalf of Intel on this matter. -- ___ Openembedded-core mailing list Openembedded-core@lists.openembedded.org http://lists.openembedded.org/mailman/listinfo/openembedded-core
Re: [OE-core] [PATCH] lib/oe/rootfs: Fix DEBUGFS generation when using opkg
On Fri, 2016-04-22 at 14:51 +0100, Richard Purdie wrote: > When enabling extra DEBUGFS image generation with opkg, errors are seen like: > > ERROR: core-image-minimal-1.0-r0 do_rootfs: Cannot get the installed packages > list. Command > '/media/build1/poky/build/tmp/sysroots/x86_64-linux/usr/bin/opkg -f > /media/build1/poky/build/tmp/work/qemux86_64-poky-linux/core-image-minimal/1.0-r0/opkg.conf > -o > /media/build1/poky/build/tmp/work/qemux86_64-poky-linux/core-image-minimal/1.0-r0/rootfs > --force_postinstall --prefer-arch-to-version status' returned 0 and > stderr: > Collected errors: > * file_md5sum_alloc: Failed to open > file > /media/build1/poky/build/tmp/work/qemux86_64-poky-linux/core-image-minimal/1.0-r0/rootfs/etc/syslog-startup.conf.busybox: > No such file or directory. > * file_md5sum_alloc: Failed to open > file > /media/build1/poky/build/tmp/work/qemux86_64-poky-linux/core-image-minimal/1.0-r0/rootfs/etc/fstab: > No such file or directory. > > basically for all CONFFILES in the image. This is due to the file > rearranging > the rootfs generation code does. If we preserve the /etc directory, > the avoids the problem. It avoids the problem, but it does not address the root cause (IMHO). Should opkg really complain about missing configuration files? It is perhaps an edge case, but besides editing a configuration file *removing* it entirely may also be a valid user modification. The usage of opkg here is during image building, but the operation itself (opkg status) is not specific to image creation and thus should be able to handle arbitrary states of /etc and the config files in general. -- Best Regards, Patrick Ohly The content of this message is my personal opinion only and although I am an employee of Intel, the statements I make here in no way represent Intel's position on the issue, nor am I authorized to speak on behalf of Intel on this matter. -- ___ Openembedded-core mailing list Openembedded-core@lists.openembedded.org http://lists.openembedded.org/mailman/listinfo/openembedded-core
[OE-core] [PATCH 3/3] combo-layer: implement "update with history"
The core idea is that all commits get imported, including merge
commits, and joined into one big merge commit that imports the changes
from the individual components into the main branch of the combined
repository.
This is done by copying the files in each commit and removing deleted
ones, instead of trying to patch the combined repository.
The advantages of doing updates in this mode are:
- works for arbitrary upstream repos, not just those which
support conversion into a linear set of patches
- listing history shows that commits where developed
independently in the different components, instead of
artificially showing them as if they had been developed
one after the after (component "" before "", then "ccc", ...)
- bisecting becomes easier: when upstream repos only ensure consistency
when merging into their "master" branches, then those merge
commits are good candidates for test builds also in the combined
repo
- more data mining can be done, for example showing who merged a commit
and when
Selecting a subset of the files is supported, albeit with a slight
different semantic for wild card matching compared to other code paths
(/ is matched by * and ?). Empty commits get skipped because typically
they are a result of filtering (but that is not checked, so
intentionally empty commits also get skipped).
Other code paths are intentionally left unchanged, to avoid
regressions. However, the downside is that some opportunities for
refactoring (in particular regarding file filtering) were ignored.
Signed-off-by: Patrick Ohly <patrick.o...@intel.com>
---
scripts/combo-layer | 391 +++-
1 file changed, 389 insertions(+), 2 deletions(-)
diff --git a/scripts/combo-layer b/scripts/combo-layer
index 9297d59..92525ca 100755
--- a/scripts/combo-layer
+++ b/scripts/combo-layer
@@ -28,6 +28,9 @@ import subprocess
import tempfile
import ConfigParser
import re
+import copy
+import pipes
+import shutil
from collections import OrderedDict
from string import Template
@@ -653,8 +656,7 @@ def action_update(conf, args):
action_pull(conf, ['arg0'] + components)
if history:
-logger.error("update with history not implemented yet")
-sys.exit(1)
+update_with_history(conf, components, revisions, repos)
else:
update_with_patches(conf, components, revisions, repos)
@@ -888,6 +890,391 @@ def action_splitpatch(conf, args):
else:
logger.info(patch_filename)
+def update_with_history(conf, components, revisions, repos):
+'''Update all components with full history.
+
+Works by importing all commits reachable from a component's
+current head revision. If those commits are rooted in an already
+imported commit, their content gets mixed with the content of the
+combined repo of that commit (new or modified files overwritten,
+removed files removed).
+
+The last commit is an artificial merge commit that merges all the
+updated components into the combined repository.
+
+The HEAD ref only gets updated at the very end. All intermediate work
+happens in a worktree which will get garbage collected by git eventually
+after a failure.
+'''
+# Remember current HEAD and what we need to add to it.
+head = runcmd("git rev-parse HEAD").strip()
+additional_heads = {}
+
+# Track the mapping between original commit and commit in the
+# combined repo. We do not have to distinguish between components,
+# because commit hashes are different anyway. Often we can
+# skip find_revs() entirely (for example, when all new commits
+# are derived from the last imported revision).
+#
+# Using "head" (typically the merge commit) instead of the actual
+# commit for the component leads to a nicer history in the combined
+# repo.
+old2new_revs = {}
+for name in repos:
+repo = conf.repos[name]
+revision = repo['last_revision']
+if revision:
+old2new_revs[revision] = head
+
+def add_p(parents):
+'''Insert -p before each entry.'''
+parameters = []
+for p in parents:
+parameters.append('-p')
+parameters.append(p)
+return parameters
+
+# Do all intermediate work with a separate work dir and index,
+# chosen via env variables (can't use "git worktree", it is too
+# new). This is useful (no changes to current work tree unless the
+# update succeeds) and required (otherwise we end up temporarily
+# removing the combo-layer hooks that we currently use when
+# importing a new component).
+#
+# Not cleaned up after a failure at the moment.
+wdir = os.path.join(os.getcwd(), ".git", "combo-layer")
+windex = wdir + ".index"
+if os.path.isdir(wdir):
+shuti
[OE-core] [PATCH 2/3] combo-layer: runcmd() enhancements
Allow setting the environment. Due to a subprocess quirk, it must
always be set explicitly (reuses the one from the previous call if not
set, instead of falling back to os.environ).
Embedding nul characters will be useful for parsing git output more
reliably; support dumping such output a bit better.
Signed-off-by: Patrick Ohly <patrick.o...@intel.com>
---
scripts/combo-layer | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
diff --git a/scripts/combo-layer b/scripts/combo-layer
index 41d69f8..9297d59 100755
--- a/scripts/combo-layer
+++ b/scripts/combo-layer
@@ -174,7 +174,7 @@ class Configuration(object):
logger.error("ERROR: patchutils package is missing, please install
it (e.g. # apt-get install patchutils)")
sys.exit(1)
-def runcmd(cmd,destdir=None,printerr=True,out=None):
+def runcmd(cmd,destdir=None,printerr=True,out=None,env=None):
"""
execute command, raise CalledProcessError if fail
return output if succeed
@@ -186,7 +186,7 @@ def runcmd(cmd,destdir=None,printerr=True,out=None):
else:
err = os.tmpfile()
try:
-subprocess.check_call(cmd, stdout=out, stderr=err, cwd=destdir,
shell=isinstance(cmd, str))
+subprocess.check_call(cmd, stdout=out, stderr=err, cwd=destdir,
shell=isinstance(cmd, str), env=env or os.environ)
except subprocess.CalledProcessError,e:
err.seek(0)
if printerr:
@@ -195,7 +195,7 @@ def runcmd(cmd,destdir=None,printerr=True,out=None):
err.seek(0)
output = err.read()
-logger.debug("output: %s" % output )
+logger.debug("output: %s" % output.replace(chr(0), '\\0'))
return output
def action_init(conf, args):
--
2.1.4
--
___
Openembedded-core mailing list
Openembedded-core@lists.openembedded.org
http://lists.openembedded.org/mailman/listinfo/openembedded-core
[OE-core] [PATCH 0/3] combo-layer import with history
In Ostro OS we recently struggled with importing repositories that had non-linear histories and duplicated patches; traditional "combo-layer update" fails to import those because it cannot flatten the upstream changes into a linear set of patches. This enhanced combo-layer can deal with that by completely re-implementing the update action such that it copies file trees around instead of patches. This adds basically another code path. It could also be maintained separately, but as it might also be of interest to others, adding this to OE-core is probably better, even though it makes the code more complicated. Probably some serious refactoring is due at some point, but that should better be done only after defining some good test cases. The following changes since commit 1a0e56630c5c27d8899dd0979ae0b86bbe227881: utils.bbclass: note for deprecated base_contains (2016-04-29 07:53:58 +0100) are available in the git repository at: git://github.com/pohly/openembedded-core combo-layer-update-history https://github.com/pohly/openembedded-core/tree/combo-layer-update-history Patrick Ohly (3): combo-layer: dummy "update with history" combo-layer: runcmd() enhancements combo-layer: implement "update with history" scripts/combo-layer | 475 +--- 1 file changed, 449 insertions(+), 26 deletions(-) -- 2.1.4 -- ___ Openembedded-core mailing list Openembedded-core@lists.openembedded.org http://lists.openembedded.org/mailman/listinfo/openembedded-core
[OE-core] [PATCH 1/3] combo-layer: dummy "update with history"
When setting "history = True" in combo-layer.conf consistently for the
components involved in an update or using "update" together with the
"--history" command line flag, a new mode for updating will be used
that does not rely on exporting/importing patches.
A config setting is used because it should be used consistently by
everyone using the same config, without having to remember to use an
additional command line parameter.
There are no real global settings, so the setting is checked
separately for each component although the setting has to be set
consistently. This restriction could be removed later.
In practice, putting "history" into the "[DEFAULT]" section is the
easiest approach for configuring it.
The actual code changes split up action_update and the
combo-layer.conf handling in preparation for this new mode, without
implementing the mode itself.
Signed-off-by: Patrick Ohly <patrick.o...@intel.com>
---
scripts/combo-layer | 82 ++---
1 file changed, 59 insertions(+), 23 deletions(-)
diff --git a/scripts/combo-layer b/scripts/combo-layer
index 9127041..41d69f8 100755
--- a/scripts/combo-layer
+++ b/scripts/combo-layer
@@ -73,7 +73,7 @@ class Configuration(object):
else:
# Apply special type transformations for some properties.
# Type matches the RawConfigParser.get*() methods.
-types = {'signoff': 'boolean', 'update': 'boolean'}
+types = {'signoff': 'boolean', 'update': 'boolean',
'history': 'boolean'}
if name in types:
value = getattr(parser, 'get' + types[name])(section,
name)
self.repos[repo][name] = value
@@ -610,8 +610,12 @@ def action_pull(conf, args):
def action_update(conf, args):
"""
update the component repos
-generate the patch list
-apply the generated patches
+either:
+ generate the patch list
+ apply the generated patches
+or:
+ re-creates the entire component history and merges them
+ into the current branch with a merge commit
"""
components = [arg.split(':')[0] for arg in args[1:]]
revisions = {}
@@ -624,10 +628,23 @@ def action_update(conf, args):
# make sure combo repo is clean
check_repo_clean(os.getcwd())
-import uuid
-patch_dir = "patch-%s" % uuid.uuid4()
-if not os.path.exists(patch_dir):
-os.mkdir(patch_dir)
+# Check whether we keep the component histories. Must be
+# set either via --history command line parameter or consistently
+# in combo-layer.conf. Mixing modes is (currently, and probably
+# permanently because it would be complicated) not supported.
+if conf.history:
+history = True
+else:
+history = None
+for name in repos:
+repo = conf.repos[name]
+repo_history = repo.get('history', True)
+logger.error('%s: %s' % (name, repo_history))
+if history is None:
+history = repo_history
+elif history != repo_history:
+logger.error("'history' property is set inconsistently")
+sys.exit(1)
# Step 1: update the component repos
if conf.nopull:
@@ -635,6 +652,18 @@ def action_update(conf, args):
else:
action_pull(conf, ['arg0'] + components)
+if history:
+logger.error("update with history not implemented yet")
+sys.exit(1)
+else:
+update_with_patches(conf, components, revisions, repos)
+
+def update_with_patches(conf, components, revisions, repos):
+import uuid
+patch_dir = "patch-%s" % uuid.uuid4()
+if not os.path.exists(patch_dir):
+os.mkdir(patch_dir)
+
for name in repos:
revision = revisions.get(name, None)
repo = conf.repos[name]
@@ -711,6 +740,21 @@ def action_update(conf, args):
runcmd("rm -rf %s" % patch_dir)
# Step 7: commit the updated config file if it's being tracked
+commit_conf_file(conf, components)
+
+def conf_commit_msg(conf, components):
+# create the "components" string
+component_str = "all components"
+if len(components) > 0:
+# otherwise tell which components were actually changed
+component_str = ", ".join(components)
+
+# expand the template with known values
+template = Template(conf.commit_msg_template)
+msg = template.substitute(components = component_str)
+return msg
+
+def commit_conf_file(conf, components, commit=True):
relpath = os.path.relpath(conf.conffile)
try:
output = runcmd("git status --porcelain %s" % relpath, printerr=False)
@@ -718,23 +762,15 @@ def action_update(conf, args
[OE-core] Busybox syslogd/klogd and journald
Hello! Can anyone think of a reason why busybox syslogd and klogd are enabled even when using systemd's journald? journald already captures both dmesg and syslog output, the other two daemons are just plain redundant. Is it perhaps because it is unclear during compilation of busybox whether the daemons are needed? -- Best Regards, Patrick Ohly The content of this message is my personal opinion only and although I am an employee of Intel, the statements I make here in no way represent Intel's position on the issue, nor am I authorized to speak on behalf of Intel on this matter. -- ___ Openembedded-core mailing list Openembedded-core@lists.openembedded.org http://lists.openembedded.org/mailman/listinfo/openembedded-core
[OE-core] [PATCH 4/5] package_manager.py: better error handling in opkg's package listing
opkg does not return a non-zero exit code even if it found
errors. When that happens, parsing the output leads to strange
follow-up errors.
To avoid this we need to check explicitly for non-empty
stderr. Reporting only that on a failure also leads to shorter error
messages (stdout may be very large).
Signed-off-by: Patrick Ohly <patrick.o...@intel.com>
---
meta/lib/oe/package_manager.py | 15 +--
1 file changed, 9 insertions(+), 6 deletions(-)
diff --git a/meta/lib/oe/package_manager.py b/meta/lib/oe/package_manager.py
index 0d23d8b..b4b359a 100644
--- a/meta/lib/oe/package_manager.py
+++ b/meta/lib/oe/package_manager.py
@@ -468,13 +468,16 @@ class OpkgPkgsList(PkgsList):
def list_pkgs(self, format=None):
cmd = "%s %s status" % (self.opkg_cmd, self.opkg_args)
-try:
-# bb.note(cmd)
-cmd_output = subprocess.check_output(cmd,
stderr=subprocess.STDOUT, shell=True).strip()
-
-except subprocess.CalledProcessError as e:
+# opkg returns success even when it printed some
+# "Collected errors:" report to stderr. Mixing stderr into
+# stdout then leads to random failures later on when
+# parsing the output. To avoid this we need to collect both
+# output streams separately and check for empty stderr.
+p = subprocess.Popen(cmd, stdout=subprocess.PIPE,
stderr=subprocess.PIPE, shell=True)
+cmd_output, cmd_stderr = p.communicate()
+if p.returncode or cmd_stderr:
bb.fatal("Cannot get the installed packages list. Command '%s' "
- "returned %d:\n%s" % (cmd, e.returncode, e.output))
+ "returned %d and stderr:\n%s" % (cmd, p.returncode,
cmd_stderr))
return self.opkg_query(cmd_output)
--
2.1.4
--
___
Openembedded-core mailing list
Openembedded-core@lists.openembedded.org
http://lists.openembedded.org/mailman/listinfo/openembedded-core
[OE-core] [PATCH 5/5] ca-certificates: support Toybox
"mktemp -t" is deprecated and does not work when using Toybox. Replace
with something that works also with Toybox.
Signed-off-by: Patrick Ohly <patrick.o...@intel.com>
---
.../update-ca-certificates-support-Toybox.patch| 35 ++
.../ca-certificates/ca-certificates_20160104.bb| 1 +
2 files changed, 36 insertions(+)
create mode 100644
meta/recipes-support/ca-certificates/ca-certificates/update-ca-certificates-support-Toybox.patch
diff --git
a/meta/recipes-support/ca-certificates/ca-certificates/update-ca-certificates-support-Toybox.patch
b/meta/recipes-support/ca-certificates/ca-certificates/update-ca-certificates-support-Toybox.patch
new file mode 100644
index 000..b3b21e2
--- /dev/null
+++
b/meta/recipes-support/ca-certificates/ca-certificates/update-ca-certificates-support-Toybox.patch
@@ -0,0 +1,35 @@
+From 30378026d136efa779732e3f6664e2ecf461e458 Mon Sep 17 00:00:00 2001
+From: Patrick Ohly <patrick.o...@intel.com>
+Date: Thu, 17 Mar 2016 12:38:09 +0100
+Subject: [PATCH] update-ca-certificates: support Toybox
+
+"mktemp -t" is deprecated and does not work when using Toybox. Replace
+with something that works also with Toybox.
+
+Upstream-Status: Inappropriate [embedded specific]
+
+Signed-off-by: Patrick Ohly <patrick.o...@intel.com>
+---
+ sbin/update-ca-certificates | 6 +++---
+ 1 file changed, 3 insertions(+), 3 deletions(-)
+
+diff --git a/sbin/update-ca-certificates b/sbin/update-ca-certificates
+index 79c41bb..ae9e3f1 100755
+--- a/sbin/update-ca-certificates
b/sbin/update-ca-certificates
+@@ -113,9 +113,9 @@ trap cleanup 0
+
+ # Helper files. (Some of them are not simple arrays because we spawn
+ # subshells later on.)
+-TEMPBUNDLE="$(mktemp -t "${CERTBUNDLE}.tmp.XX")"
+-ADDED="$(mktemp -t "ca-certificates.tmp.XX")"
+-REMOVED="$(mktemp -t "ca-certificates.tmp.XX")"
++TEMPBUNDLE="$(mktemp -p${TMPDIR:-/tmp} "${CERTBUNDLE}.tmp.XX")"
++ADDED="$(mktemp -p${TMPDIR:-/tmp} "ca-certificates.tmp.XX")"
++REMOVED="$(mktemp -p${TMPDIR:-/tmp} "ca-certificates.tmp.XX")"
+
+ # Adds a certificate to the list of trusted ones. This includes a symlink
+ # in /etc/ssl/certs to the certificate file and its inclusion into the
+--
+2.1.4
+
diff --git a/meta/recipes-support/ca-certificates/ca-certificates_20160104.bb
b/meta/recipes-support/ca-certificates/ca-certificates_20160104.bb
index 4266926..e0f1939 100644
--- a/meta/recipes-support/ca-certificates/ca-certificates_20160104.bb
+++ b/meta/recipes-support/ca-certificates/ca-certificates_20160104.bb
@@ -17,6 +17,7 @@ SRC_URI =
"git://anonscm.debian.org/collab-maint/ca-certificates.git \
file://0001-update-ca-certificates-remove-c-rehash.patch \
file://0002-update-ca-certificates-use-SYSROOT.patch \
file://0001-update-ca-certificates-don-t-use-Debianisms-in-run-p.patch \
+ file://update-ca-certificates-support-Toybox.patch \
file://default-sysroot.patch \
file://sbindir.patch"
--
2.1.4
--
___
Openembedded-core mailing list
Openembedded-core@lists.openembedded.org
http://lists.openembedded.org/mailman/listinfo/openembedded-core
[OE-core] [PATCH 2/5] ncurses: reorder PACKAGES
Having ncurses-terminfo-base before ncurses-terminfo is currently irrelevant
because the current file lists are completely disjunct. However, when building
"stateless" via a .bbappend, the content of curses-terminfo-base also needs
to live under /usr/share and then it becomes important that files
belonging to ncurses-terminfo-base are checked first.
Signed-off-by: Patrick Ohly <patrick.o...@intel.com>
---
meta/recipes-core/ncurses/ncurses.inc | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/meta/recipes-core/ncurses/ncurses.inc
b/meta/recipes-core/ncurses/ncurses.inc
index acd2579..ff0117b 100644
--- a/meta/recipes-core/ncurses/ncurses.inc
+++ b/meta/recipes-core/ncurses/ncurses.inc
@@ -269,8 +269,8 @@ BBCLASSEXTEND = "native nativesdk"
PACKAGES += " \
${PN}-tools \
- ${PN}-terminfo \
${PN}-terminfo-base \
+ ${PN}-terminfo \
"
FILES_${PN} = "\
--
2.1.4
--
___
Openembedded-core mailing list
Openembedded-core@lists.openembedded.org
http://lists.openembedded.org/mailman/listinfo/openembedded-core
[OE-core] [PATCH 3/5] systemd: make systemd-serialgetty optional
Some distros may prefer to use the upstream systemd support for
starting getty on serial ports. This is now possible by adding
"serial-getty-generator" to PACKAGECONFIG.
The default is unchanged, i.e. systemd's own serial-getty@.service
file does not get packaged and instead systemd-serialgetty is pulled
into images via RRECOMMENDS.
Signed-off-by: Patrick Ohly <patrick.o...@intel.com>
---
meta/recipes-core/systemd/systemd_229.bb | 14 +++---
1 file changed, 11 insertions(+), 3 deletions(-)
diff --git a/meta/recipes-core/systemd/systemd_229.bb
b/meta/recipes-core/systemd/systemd_229.bb
index 309a4c3..c23c749 100644
--- a/meta/recipes-core/systemd/systemd_229.bb
+++ b/meta/recipes-core/systemd/systemd_229.bb
@@ -96,6 +96,11 @@ PACKAGECONFIG ??= "xz \
PACKAGECONFIG_remove_libc-musl = "selinux"
PACKAGECONFIG_remove_libc-musl = "smack"
+# Use the upstream systemd serial-getty@.service and rely on
+# systemd-getty-generator instead of using the OE-core specific
+# systemd-serialgetty.bb - not enabled by default.
+PACKAGECONFIG[serial-getty-generator] = ""
+
PACKAGECONFIG[journal-upload] = "--enable-libcurl,--disable-libcurl,curl"
# Sign the journal for anti-tampering
PACKAGECONFIG[gcrypt] = "--enable-gcrypt,--disable-gcrypt,libgcrypt"
@@ -204,8 +209,10 @@ do_configure_prepend() {
do_install() {
autotools_do_install
install -d ${D}/${base_sbindir}
- # Provided by a separate recipe
- rm ${D}${systemd_unitdir}/system/serial-getty* -f
+ if ${@bb.utils.contains('PACKAGECONFIG', 'serial-getty-generator',
'false', 'true', d)}; then
+ # Provided by a separate recipe
+ rm ${D}${systemd_unitdir}/system/serial-getty* -f
+ fi
# Provide support for initramfs
[ ! -e ${D}/init ] && ln -s ${rootlibexecdir}/systemd/systemd ${D}/init
@@ -455,7 +462,8 @@ FILES_${PN}-dev += "${base_libdir}/security/*.la
${datadir}/dbus-1/interfaces/ $
RDEPENDS_${PN} += "kmod dbus util-linux-mount udev (= ${EXTENDPKGV})"
RDEPENDS_${PN} += "volatile-binds update-rc.d"
-RRECOMMENDS_${PN} += "systemd-serialgetty systemd-vconsole-setup \
+RRECOMMENDS_${PN} += "${@bb.utils.contains('PACKAGECONFIG',
'serial-getty-generator', '', 'systemd-serialgetty', d)} \
+ systemd-vconsole-setup \
systemd-extra-utils \
systemd-compat-units udev-hwdb \
util-linux-agetty util-linux-fsck e2fsprogs-e2fsck \
--
2.1.4
--
___
Openembedded-core mailing list
Openembedded-core@lists.openembedded.org
http://lists.openembedded.org/mailman/listinfo/openembedded-core
[OE-core] [PATCH 1/5] bluez5.inc: remove obsolete workaround
Bluez 5.37 itself correctly installs bluetooth.conf, and honors
the path settings in dbus-1.pc.
Removing the obsolete workaround is necessary for compiling
"stateless" (= read-only system configuration moved out of /etc).
Signed-off-by: Patrick Ohly <patrick.o...@intel.com>
---
meta/recipes-connectivity/bluez5/bluez5.inc | 2 --
1 file changed, 2 deletions(-)
diff --git a/meta/recipes-connectivity/bluez5/bluez5.inc
b/meta/recipes-connectivity/bluez5/bluez5.inc
index 5232523..a508229 100644
--- a/meta/recipes-connectivity/bluez5/bluez5.inc
+++ b/meta/recipes-connectivity/bluez5/bluez5.inc
@@ -60,8 +60,6 @@ do_install_append() {
install -m 0644 ${S}/profiles/input/input.conf
${D}/${sysconfdir}/bluetooth/
fi
- install -m 0644 ${S}/src/bluetooth.conf
${D}/${sysconfdir}/dbus-1/system.d/
-
# Install desired tools that upstream leaves in build area
for f in ${NOINST_TOOLS} ; do
install -m 755 ${B}/$f ${D}/${bindir}
--
2.1.4
--
___
Openembedded-core mailing list
Openembedded-core@lists.openembedded.org
http://lists.openembedded.org/mailman/listinfo/openembedded-core
[OE-core] [PATCH 0/5] misc enhancements
I've worked on making Ostro OS support Toybox and "stateless" configuration, i.e. running with empty /etc and reserving that for local configuration changes. That work depends on several enhancements in OE-core. "package_manager.py: better error handling in opkg's package listing" is not needed anymore, but helped debug a packaging problem. Please pick whatever you deem suitable for OE-core at this time, I'll keep maintaining the rest and can re-submit again later. The following changes since commit d60806e56aed2f62f6a0e030a564f7fdc4a1314d: build-appliance-image: Exclude DDATETIME from task signature (2016-04-03 15:51:10 +0100) are available in the git repository at: git://github.com/pohly/openembedded-core stateless https://github.com/pohly/openembedded-core/tree/stateless Patrick Ohly (5): bluez5.inc: remove obsolete workaround ncurses: reorder PACKAGES systemd: make systemd-serialgetty optional package_manager.py: better error handling in opkg's package listing ca-certificates: support Toybox meta/lib/oe/package_manager.py | 15 ++ meta/recipes-connectivity/bluez5/bluez5.inc| 2 -- meta/recipes-core/ncurses/ncurses.inc | 2 +- meta/recipes-core/systemd/systemd_229.bb | 14 +++-- .../update-ca-certificates-support-Toybox.patch| 35 ++ .../ca-certificates/ca-certificates_20160104.bb| 1 + 6 files changed, 57 insertions(+), 12 deletions(-) create mode 100644 meta/recipes-support/ca-certificates/ca-certificates/update-ca-certificates-support-Toybox.patch -- 2.1.4 -- ___ Openembedded-core mailing list Openembedded-core@lists.openembedded.org http://lists.openembedded.org/mailman/listinfo/openembedded-core
[OE-core] [PATCH] buildhistory.bbclass: create image directory when needed
buildhistory_get_imageinfo() assumed that the buildhistory directory
for the image had already been created earlier. That assumption is not
true for special images (like the virtual swupd images from
meta-swupd) where the entire traditional do_rootfs/do_image is
skipped.
Creating files-in-image.txt still makes sense for such images, so
support them by creating the directory also in
buildhistory_get_imageinfo().
Signed-off-by: Patrick Ohly <patrick.o...@intel.com>
---
meta/classes/buildhistory.bbclass | 1 +
1 file changed, 1 insertion(+)
diff --git a/meta/classes/buildhistory.bbclass
b/meta/classes/buildhistory.bbclass
index 0b3b7ce..108275a 100644
--- a/meta/classes/buildhistory.bbclass
+++ b/meta/classes/buildhistory.bbclass
@@ -504,6 +504,7 @@ buildhistory_get_imageinfo() {
return
fi
+mkdir -p ${BUILDHISTORY_DIR_IMAGE}
buildhistory_list_files ${IMAGE_ROOTFS}
${BUILDHISTORY_DIR_IMAGE}/files-in-image.txt
# Collect files requested in BUILDHISTORY_IMAGE_FILES
--
2.1.4
--
___
Openembedded-core mailing list
Openembedded-core@lists.openembedded.org
http://lists.openembedded.org/mailman/listinfo/openembedded-core
[OE-core] [PATCH 1/3] sanity.bbclass: expand error messages for version checks
The ${WORKDIR} variable was not getting expanded (anymore?), leading to
less helpful error messages like:
Exception: NotImplementedError: Your version of bblayers.conf has the wrong
LCONF_VERSION (has 7, expecting 8).
Please compare your file against bblayers.conf.sample and merge any changes
before continuing.
"meld conf/bblayers.conf ${COREBASE}/meta*/conf/bblayers.conf.sample"
is a good way to visualise the changes.
ERROR: Execution of event handler 'check_sanity_eventhandler' failed
After adding expansion, embedding ${LCONF_VERSION} and ${LAYER_CONF_VERSION}
in the error message seems a bit more readable and consistent.
Signed-off-by: Patrick Ohly <patrick.o...@intel.com>
---
meta/classes/sanity.bbclass | 7 +--
1 file changed, 5 insertions(+), 2 deletions(-)
diff --git a/meta/classes/sanity.bbclass b/meta/classes/sanity.bbclass
index 4873e64..334e362 100644
--- a/meta/classes/sanity.bbclass
+++ b/meta/classes/sanity.bbclass
@@ -48,6 +48,7 @@ Matching the version numbers will remove this message.
\"meld conf/local.conf ${COREBASE}/meta*/conf/local.conf.sample\"
is a good way to visualise the changes."
+failmsg = d.expand(failmsg)
raise NotImplementedError(failmsg)
}
@@ -66,6 +67,7 @@ Matching the version numbers will remove this message.
\"meld conf/site.conf ${COREBASE}/meta*/conf/site.conf.sample\"
is a good way to visualise the changes."
+failmsg = d.expand(failmsg)
raise NotImplementedError(failmsg)
}
@@ -76,11 +78,12 @@ python oecore_update_bblayers() {
current_lconf = int(d.getVar('LCONF_VERSION', True))
lconf_version = int(d.getVar('LAYER_CONF_VERSION', True))
-failmsg = """Your version of bblayers.conf has the wrong LCONF_VERSION
(has %s, expecting %s).
+failmsg = """Your version of bblayers.conf has the wrong LCONF_VERSION
(has ${LCONF_VERSION}, expecting ${LAYER_CONF_VERSION}).
Please compare your file against bblayers.conf.sample and merge any changes
before continuing.
"meld conf/bblayers.conf ${COREBASE}/meta*/conf/bblayers.conf.sample"
-is a good way to visualise the changes.""" % (current_lconf, lconf_version)
+is a good way to visualise the changes."""
+failmsg = d.expand(failmsg)
if not current_lconf:
raise NotImplementedError(failmsg)
--
2.1.4
--
___
Openembedded-core mailing list
Openembedded-core@lists.openembedded.org
http://lists.openembedded.org/mailman/listinfo/openembedded-core
[OE-core] [PATCH 3/3] sanity.bbclass: allow customizing config file update error messages
The default file pattern may be amiguous and "meld" might not always
be the preferred tool, so allow distros to override those parts of the
error messages.
Signed-off-by: Patrick Ohly <patrick.o...@intel.com>
---
meta/classes/sanity.bbclass | 11 ---
1 file changed, 8 insertions(+), 3 deletions(-)
diff --git a/meta/classes/sanity.bbclass b/meta/classes/sanity.bbclass
index 2539bd8..e72a007 100644
--- a/meta/classes/sanity.bbclass
+++ b/meta/classes/sanity.bbclass
@@ -34,6 +34,9 @@ BBLAYERS_CONF_UPDATE_FUNCS += " \
conf/site.conf:SCONF_VERSION:SITE_CONF_VERSION:oecore_update_siteconf \
"
+SANITY_DIFF_TOOL ?= "meld"
+
+SANITY_LOCALCONF_SAMPLE ?= "${COREBASE}/meta*/conf/local.conf.sample"
python oecore_update_localconf() {
# Check we are using a valid local.conf
current_conf = d.getVar('CONF_VERSION', True)
@@ -45,7 +48,7 @@ files and merge any changes before continuing.
Matching the version numbers will remove this message.
-\"meld conf/local.conf ${COREBASE}/meta*/conf/local.conf.sample\"
+\"${SANITY_DIFF_TOOL} conf/local.conf ${SANITY_LOCALCONF_SAMPLE}\"
is a good way to visualise the changes."
failmsg = d.expand(failmsg)
@@ -53,6 +56,7 @@ is a good way to visualise the changes."
raise NotImplementedError(failmsg)
}
+SANITY_SITECONF_SAMPLE ?= "${COREBASE}/meta*/conf/site.conf.sample"
python oecore_update_siteconf() {
# If we have a site.conf, check it's valid
current_sconf = d.getVar('SCONF_VERSION', True)
@@ -64,7 +68,7 @@ files and merge any changes before continuing.
Matching the version numbers will remove this message.
-\"meld conf/site.conf ${COREBASE}/meta*/conf/site.conf.sample\"
+\"${SANITY_DIFF_TOOL} conf/site.conf ${SANITY_SITECONF_SAMPLE}\"
is a good way to visualise the changes."
failmsg = d.expand(failmsg)
@@ -72,6 +76,7 @@ is a good way to visualise the changes."
raise NotImplementedError(failmsg)
}
+SANITY_BBLAYERCONF_SAMPLE ?= "${COREBASE}/meta*/conf/bblayers.conf.sample"
python oecore_update_bblayers() {
# bblayers.conf is out of date, so see if we can resolve that
@@ -80,7 +85,7 @@ python oecore_update_bblayers() {
failmsg = """Your version of bblayers.conf has the wrong LCONF_VERSION
(has ${LCONF_VERSION}, expecting ${LAYER_CONF_VERSION}).
Please compare your file against bblayers.conf.sample and merge any changes
before continuing.
-"meld conf/bblayers.conf ${COREBASE}/meta*/conf/bblayers.conf.sample"
+"${SANITY_DIFF_TOOL} conf/bblayers.conf ${SANITY_BBLAYERCONF_SAMPLE}"
is a good way to visualise the changes."""
failmsg = d.expand(failmsg)
--
2.1.4
--
___
Openembedded-core mailing list
Openembedded-core@lists.openembedded.org
http://lists.openembedded.org/mailman/listinfo/openembedded-core
[OE-core] [PATCH 2/3] sanity.bbclass: fix success message when config file was updated
The code now iterates over different config files, but always printed a
message about conf/bblayers.conf for each file.
Signed-off-by: Patrick Ohly <patrick.o...@intel.com>
---
meta/classes/sanity.bbclass | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/meta/classes/sanity.bbclass b/meta/classes/sanity.bbclass
index 334e362..2539bd8 100644
--- a/meta/classes/sanity.bbclass
+++ b/meta/classes/sanity.bbclass
@@ -559,7 +559,7 @@ def sanity_check_conffiles(status, d):
success = False
status.addresult(e.msg)
if success:
-bb.note("Your conf/bblayers.conf has been automatically
updated.")
+bb.note("Your %s file has been automatically updated." %
conffile)
status.reparse = True
def sanity_handle_abichanges(status, d):
--
2.1.4
--
___
Openembedded-core mailing list
Openembedded-core@lists.openembedded.org
http://lists.openembedded.org/mailman/listinfo/openembedded-core
[OE-core] [PATCH] image.bbclass: track ROOTFS_POSTUNINSTALL_COMMAND in do_rootfs vardeps
The list of variables influencing do_rootfs was not updated when
introducing ROOTFS_POSTUNINSTALL_COMMAND. As a result, making changes
in commands listed there or the variables they depend on did not trigger
a re-run of do_rootfs.
Signed-off-by: Patrick Ohly <patrick.o...@intel.com>
---
meta/classes/image.bbclass | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/meta/classes/image.bbclass b/meta/classes/image.bbclass
index c61d814..3ab432e 100644
--- a/meta/classes/image.bbclass
+++ b/meta/classes/image.bbclass
@@ -98,7 +98,7 @@ do_rootfs[depends] += " \
do_rootfs[recrdeptask] += "do_packagedata"
def rootfs_command_variables(d):
-return
['ROOTFS_POSTPROCESS_COMMAND','ROOTFS_PREPROCESS_COMMAND','ROOTFS_POSTINSTALL_COMMAND','OPKG_PREPROCESS_COMMANDS','OPKG_POSTPROCESS_COMMANDS','IMAGE_POSTPROCESS_COMMAND',
+return
['ROOTFS_POSTPROCESS_COMMAND','ROOTFS_PREPROCESS_COMMAND','ROOTFS_POSTINSTALL_COMMAND','ROOTFS_POSTUNINSTALL_COMMAND','OPKG_PREPROCESS_COMMANDS','OPKG_POSTPROCESS_COMMANDS','IMAGE_POSTPROCESS_COMMAND',
'IMAGE_PREPROCESS_COMMAND','ROOTFS_POSTPROCESS_COMMAND','RPM_PREPROCESS_COMMANDS','RPM_POSTPROCESS_COMMANDS']
python () {
--
2.1.4
--
___
Openembedded-core mailing list
Openembedded-core@lists.openembedded.org
http://lists.openembedded.org/mailman/listinfo/openembedded-core
[OE-core] [PATCH v2] image-buildinfo.bbclass: fix performance problems
Inheriting image-buildinfo.bbclass primarily slowed down image
building for two reasons:
1. The content of the shell command "buildinfo" gets expanded
multiple times, each time again checking the state of all
layers.
2. When expanded as part of the actual image creation, git
is invoked under pseudo, which makes the check quite a bit
slower (from a few seconds to a minute with many layers).
To fix this, buildinfo now is a Python method which calls the checks
only when really executed. Pseudo is told to unload itself when
starting git.
In addition, "git diff" is invoked with "--quiet", which avoids
producing output that is just getting thrown away. As before, any kind
of problem or output causes the layer to be marked as "modified".
[Revision 2 of the change with some dead code removed]
Signed-off-by: Patrick Ohly <patrick.o...@intel.com>
---
meta/classes/image-buildinfo.bbclass | 35 ++-
1 file changed, 22 insertions(+), 13 deletions(-)
diff --git a/meta/classes/image-buildinfo.bbclass
b/meta/classes/image-buildinfo.bbclass
index 5b738ae..197b242 100644
--- a/meta/classes/image-buildinfo.bbclass
+++ b/meta/classes/image-buildinfo.bbclass
@@ -26,12 +26,17 @@ def image_buildinfo_outputvars(vars, listvars, d):
# Gets git branch's status (clean or dirty)
def get_layer_git_status(path):
-f = os.popen("cd %s; git diff --stat 2>&1 | tail -n 1" % path)
-data = f.read()
-if f.close() is None:
-if len(data) != 0:
-return "-- modified"
-return ""
+import subprocess
+try:
+subprocess.check_output("cd %s; PSEUDO_UNLOAD=1 git diff --quiet
--no-ext-diff" % path,
+shell=True,
+stderr=subprocess.STDOUT)
+return ""
+except subprocess.CalledProcessError, ex:
+# Silently treat errors as "modified", without checking for the
+# (expected) return code 1 in a modified git repo. For example, we get
+# output and a 129 return code when a layer isn't a git repo at all.
+return "-- modified"
# Returns layer revisions along with their respective status
def get_layer_revs(d):
@@ -53,17 +58,21 @@ def buildinfo_target(d):
return image_buildinfo_outputvars(vars, listvars, d)
# Write build information to target filesystem
-buildinfo () {
-cat > ${IMAGE_ROOTFS}${sysconfdir}/build << END
+python buildinfo () {
+with open(d.expand('${IMAGE_ROOTFS}${sysconfdir}/build'), 'w') as build:
+build.writelines((
+'''---
Build Configuration: |
---
-${@buildinfo_target(d)}
+''',
+buildinfo_target(d),
+'''
---
-Layer Revisions: |
+Layer Revisions: |
---
-${@get_layer_revs(d)}
-END
+''',
+get_layer_revs(d)
+ ))
}
IMAGE_PREPROCESS_COMMAND += "buildinfo;"
--
2.1.4
--
___
Openembedded-core mailing list
Openembedded-core@lists.openembedded.org
http://lists.openembedded.org/mailman/listinfo/openembedded-core
[OE-core] [PATCH] image-buildinfo.bbclass: fix performance problems
Inheriting image-buildinfo.bbclass primarily slowed down image
building for two reasons:
1. The content of the shell command "buildinfo" gets expanded
multiple times, each time again checking the state of all
layers.
2. When expanded as part of the actual image creation, git
is invoked under pseudo, which makes the check quite a bit
slower (from a few seconds to a minute with many layers).
To fix this, buildinfo now is a Python method which calls the checks
only when really executed. Pseudo is told to unload itself when
starting git.
In addition, "git diff" is invoked with "--quiet", which avoids
producing output that is just getting thrown away. As before, any kind
of problem or output causes the layer to be marked as "modified".
Signed-off-by: Patrick Ohly <patrick.o...@intel.com>
---
meta/classes/image-buildinfo.bbclass | 39
1 file changed, 26 insertions(+), 13 deletions(-)
diff --git a/meta/classes/image-buildinfo.bbclass
b/meta/classes/image-buildinfo.bbclass
index 5b738ae..408b5b8 100644
--- a/meta/classes/image-buildinfo.bbclass
+++ b/meta/classes/image-buildinfo.bbclass
@@ -26,12 +26,21 @@ def image_buildinfo_outputvars(vars, listvars, d):
# Gets git branch's status (clean or dirty)
def get_layer_git_status(path):
-f = os.popen("cd %s; git diff --stat 2>&1 | tail -n 1" % path)
-data = f.read()
-if f.close() is None:
-if len(data) != 0:
-return "-- modified"
-return ""
+import subprocess
+import copy
+import os
+try:
+env = copy.deepcopy(os.environ)
+subprocess.check_output("cd %s; PSEUDO_UNLOAD=1 git diff --quiet
--no-ext-diff" % path,
+shell=True,
+stderr=subprocess.STDOUT,
+env=env)
+return ""
+except subprocess.CalledProcessError, ex:
+# Silently treat errors as "modified", without checking for the
+# (expected) return code 1 in a modified git repo. For example, we get
+# output and a 129 return code when a layer isn't a git repo at all.
+return "-- modified"
# Returns layer revisions along with their respective status
def get_layer_revs(d):
@@ -53,17 +62,21 @@ def buildinfo_target(d):
return image_buildinfo_outputvars(vars, listvars, d)
# Write build information to target filesystem
-buildinfo () {
-cat > ${IMAGE_ROOTFS}${sysconfdir}/build << END
+python buildinfo () {
+with open(d.expand('${IMAGE_ROOTFS}${sysconfdir}/build'), 'w') as build:
+build.writelines((
+'''---
Build Configuration: |
---
-${@buildinfo_target(d)}
+''',
+buildinfo_target(d),
+'''
---
-Layer Revisions: |
+Layer Revisions: |
---
-${@get_layer_revs(d)}
-END
+''',
+get_layer_revs(d)
+ ))
}
IMAGE_PREPROCESS_COMMAND += "buildinfo;"
--
2.1.4
--
___
Openembedded-core mailing list
Openembedded-core@lists.openembedded.org
http://lists.openembedded.org/mailman/listinfo/openembedded-core
[OE-core] [RFC PATCH] systemd: make systemd-serialgetty optional
Some distros may prefer to use the upstream systemd support for
starting getty on serial ports. This is now possible by adding
"serial-getty-generator" to PACKAGECONFIG.
The default is unchanged, i.e. systemd's own serial-getty@.service
file does not get packaged and instead systemd-serialgetty is pulled
into images via RRECOMMENDS.
Signed-off-by: Patrick Ohly <patrick.o...@intel.com>
---
meta/recipes-core/systemd/systemd_229.bb | 15 ---
1 file changed, 12 insertions(+), 3 deletions(-)
diff --git a/meta/recipes-core/systemd/systemd_229.bb
b/meta/recipes-core/systemd/systemd_229.bb
index cd48360..3c4d877 100644
--- a/meta/recipes-core/systemd/systemd_229.bb
+++ b/meta/recipes-core/systemd/systemd_229.bb
@@ -148,6 +148,11 @@ PACKAGECONFIG[lz4] = "--enable-lz4,--disable-lz4,lz4"
PACKAGECONFIG[xz] = "--enable-xz,--disable-xz,xz"
PACKAGECONFIG[zlib] = "--enable-zlib,--disable-zlib,zlib"
+# Use the upstream systemd serial-getty@.service and rely on
+# systemd-getty-generator instead of using the OE-core specific
+# systemd-serialgetty.bb - not enabled by default.
+PACKAGECONFIG[serial-getty-generator] = ""
+
CACHED_CONFIGUREVARS += "ac_cv_path_KILL=${base_bindir}/kill"
CACHED_CONFIGUREVARS += "ac_cv_path_KMOD=${base_bindir}/kmod"
CACHED_CONFIGUREVARS += "ac_cv_path_QUOTACHECK=${sbindir}/quotacheck"
@@ -201,8 +206,10 @@ do_configure_prepend() {
do_install() {
autotools_do_install
install -d ${D}/${base_sbindir}
- # Provided by a separate recipe
- rm ${D}${systemd_unitdir}/system/serial-getty* -f
+ if ${@bb.utils.contains('PACKAGECONFIG', 'serial-getty-generator',
'false', 'true', d)}; then
+ # Provided by a separate recipe
+ rm ${D}${systemd_unitdir}/system/serial-getty* -f
+ fi
# Provide support for initramfs
[ ! -e ${D}/init ] && ln -s ${rootlibexecdir}/systemd/systemd ${D}/init
@@ -439,7 +446,9 @@ FILES_${PN}-dev += "${base_libdir}/security/*.la
${datadir}/dbus-1/interfaces/ $
RDEPENDS_${PN} += "kmod dbus util-linux-mount udev (= ${EXTENDPKGV})"
RDEPENDS_${PN} += "volatile-binds update-rc.d"
-RRECOMMENDS_${PN} += "systemd-serialgetty systemd-vconsole-setup \
+RRECOMMENDS_${PN} += " \
+ ${@bb.utils.contains('PACKAGECONFIG',
'serial-getty-generator', '', 'systemd-serialgetty', d)} \
+ systemd-vconsole-setup \
systemd-extra-utils \
systemd-compat-units udev-hwdb \
util-linux-agetty util-linux-fsck e2fsprogs-e2fsck \
--
2.1.4
--
___
Openembedded-core mailing list
Openembedded-core@lists.openembedded.org
http://lists.openembedded.org/mailman/listinfo/openembedded-core
[OE-core] systemd-serial-getty obsolete?
Hello! systemd-serialgetty.bb was introduced in OE-core together with systemd 1.96 and probably goes back even further (copied from meta-systemd). It explicitly instantiates serial-getty@.service for serial ports specified by SERIAL_CONSOLES. I'm wondering whether that is still needed. systemd also has a generator [1] that can activate serial-getty based on kernel boot parameters [2]. Is systemd-serialgetty.bb perhaps kept around because the "console" parameters added to APPEND are (perhaps intentionally) not the same as SERIAL_CONSOLES? [1] https://www.freedesktop.org/software/systemd/man/systemd-getty-generator.html [2] http://0pointer.de/blog/projects/serial-console.html -- Best Regards, Patrick Ohly The content of this message is my personal opinion only and although I am an employee of Intel, the statements I make here in no way represent Intel's position on the issue, nor am I authorized to speak on behalf of Intel on this matter. -- ___ Openembedded-core mailing list Openembedded-core@lists.openembedded.org http://lists.openembedded.org/mailman/listinfo/openembedded-core
[OE-core] [PATCH] image.bbclass: fix incomplete .rootfs customization
The patch for making the .rootfs configurable was incomplete: in the
python create_symlinks() method the new variable must be expanded
explicitly.
Not doing so broke the symlink creation and that led to hard build
failures in image types depending on the boot-directdisk.bbclass (like
qcow2) because the build_boot_dd() method relied on the symlink.
Signed-off-by: Patrick Ohly <patrick.o...@intel.com>
---
meta/classes/image.bbclass | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/meta/classes/image.bbclass b/meta/classes/image.bbclass
index 081a0b3..8b6c30b 100644
--- a/meta/classes/image.bbclass
+++ b/meta/classes/image.bbclass
@@ -474,7 +474,7 @@ python create_symlinks() {
manifest_name = d.getVar('IMAGE_MANIFEST', True)
taskname = d.getVar("BB_CURRENTTASK", True)
subimages = (d.getVarFlag("do_" + taskname, 'subimages', False) or
"").split()
-imgsuffix = d.getVarFlag("do_" + taskname, 'imgsuffix', True) or
"${IMAGE_NAME_SUFFIX}."
+imgsuffix = d.getVarFlag("do_" + taskname, 'imgsuffix', True) or
d.expand("${IMAGE_NAME_SUFFIX}.")
os.chdir(deploy_dir)
if not link_name:
--
2.1.4
--
___
Openembedded-core mailing list
Openembedded-core@lists.openembedded.org
http://lists.openembedded.org/mailman/listinfo/openembedded-core
[OE-core] [PATCH 1/2] image creation: allow overriding .rootfs suffix
By default, the image file name contains ".rootfs" to distinguish the
image file from other files created during image building. However,
for certain image types (for example, .hddimg) the ".rootfs" suffix is
redundant because the type suffix alone already uniquely identifies
the main image file (core-image-minimal-intel-corei7-64.hddimg instead
of core-image-minimal-intel-corei7-64.rootfs.hddimg).
With this change, distros that prefer the shorter image name can
override the .rootfs suffix like this:
IMAGE_NAME_ROOTFS = "${@ '' if else
'${IMAGE_NAME_ROOTFS_DEFAULT}'}"
The exact logic when to remove the extra suffix depends on the distro
and how it enables its own image type.
Signed-off-by: Patrick Ohly <patrick.o...@intel.com>
---
meta/classes/image.bbclass | 4 +--
meta/classes/image_types.bbclass | 63 ++--
2 files changed, 36 insertions(+), 31 deletions(-)
diff --git a/meta/classes/image.bbclass b/meta/classes/image.bbclass
index 02612ad..081a0b3 100644
--- a/meta/classes/image.bbclass
+++ b/meta/classes/image.bbclass
@@ -387,7 +387,7 @@ python () {
subimages.append(realt + "." + ctype)
if realt not in alltypes:
-cmds.append(localdata.expand("\trm ${IMAGE_NAME}.rootfs.${type}"))
+cmds.append(localdata.expand("\trm
${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.${type}"))
else:
subimages.append(realt)
@@ -474,7 +474,7 @@ python create_symlinks() {
manifest_name = d.getVar('IMAGE_MANIFEST', True)
taskname = d.getVar("BB_CURRENTTASK", True)
subimages = (d.getVarFlag("do_" + taskname, 'subimages', False) or
"").split()
-imgsuffix = d.getVarFlag("do_" + taskname, 'imgsuffix', True) or ".rootfs."
+imgsuffix = d.getVarFlag("do_" + taskname, 'imgsuffix', True) or
"${IMAGE_NAME_SUFFIX}."
os.chdir(deploy_dir)
if not link_name:
diff --git a/meta/classes/image_types.bbclass b/meta/classes/image_types.bbclass
index db8cb37..22719ad 100644
--- a/meta/classes/image_types.bbclass
+++ b/meta/classes/image_types.bbclass
@@ -1,3 +1,8 @@
+# IMAGE_NAME is the base name for everything produced when building images.
+# The actual image that contains the rootfs has an additional suffix (.rootfs
+# by default) followed by additional suffices which describe the format (.ext4,
+# .ext4.xz, etc.).
+IMAGE_NAME_SUFFIX ??= ".rootfs"
# The default aligment of the size of the rootfs is set to 1KiB. In case
# you're using the SD card emulation of a QEMU system simulator you may
@@ -36,9 +41,9 @@ XZ_INTEGRITY_CHECK ?= "crc32"
XZ_THREADS ?= "-T 0"
JFFS2_SUM_EXTRA_ARGS ?= ""
-IMAGE_CMD_jffs2 = "mkfs.jffs2 --root=${IMAGE_ROOTFS} --faketime
--output=${DEPLOY_DIR_IMAGE}/${IMAGE_NAME}.rootfs.jffs2 ${EXTRA_IMAGECMD}"
+IMAGE_CMD_jffs2 = "mkfs.jffs2 --root=${IMAGE_ROOTFS} --faketime
--output=${DEPLOY_DIR_IMAGE}/${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.jffs2
${EXTRA_IMAGECMD}"
-IMAGE_CMD_cramfs = "mkfs.cramfs ${IMAGE_ROOTFS}
${DEPLOY_DIR_IMAGE}/${IMAGE_NAME}.rootfs.cramfs ${EXTRA_IMAGECMD}"
+IMAGE_CMD_cramfs = "mkfs.cramfs ${IMAGE_ROOTFS}
${DEPLOY_DIR_IMAGE}/${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.cramfs ${EXTRA_IMAGECMD}"
oe_mkext234fs () {
fstype=$1
@@ -58,8 +63,8 @@ oe_mkext234fs () {
eval COUNT=\"$MIN_COUNT\"
fi
# Create a sparse image block
- dd if=/dev/zero of=${DEPLOY_DIR_IMAGE}/${IMAGE_NAME}.rootfs.$fstype
seek=$ROOTFS_SIZE count=$COUNT bs=1024
- mkfs.$fstype -F $extra_imagecmd
${DEPLOY_DIR_IMAGE}/${IMAGE_NAME}.rootfs.$fstype -d ${IMAGE_ROOTFS}
+ dd if=/dev/zero
of=${DEPLOY_DIR_IMAGE}/${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.$fstype
seek=$ROOTFS_SIZE count=$COUNT bs=1024
+ mkfs.$fstype -F $extra_imagecmd
${DEPLOY_DIR_IMAGE}/${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.$fstype -d ${IMAGE_ROOTFS}
}
IMAGE_CMD_ext2 = "oe_mkext234fs ext2 ${EXTRA_IMAGECMD}"
@@ -69,16 +74,16 @@ IMAGE_CMD_ext4 = "oe_mkext234fs ext4 ${EXTRA_IMAGECMD}"
MIN_BTRFS_SIZE ?= "16384"
IMAGE_CMD_btrfs () {
if [ ${ROOTFS_SIZE} -gt ${MIN_BTRFS_SIZE} ]; then
- dd if=/dev/zero
of=${DEPLOY_DIR_IMAGE}/${IMAGE_NAME}.rootfs.btrfs count=${ROOTFS_SIZE} bs=1024
- mkfs.btrfs ${EXTRA_IMAGECMD} -r ${IMAGE_ROOTFS}
${DEPLOY_DIR_IMAGE}/${IMAGE_NAME}.rootfs.btrfs
+ dd if=/dev/zero
of=${DEPLOY_DIR_IMAGE}/${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.btrfs
count=${ROOTFS_SIZE} bs=1024
+ mkfs.btrfs ${EXTRA_IMAGECMD} -r ${IMAGE_ROOTFS}
${DEPLOY_DIR_IMAGE}/${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.btrfs
else
bbfatal "Rootfs is too small for BTRFS (Rootfs Actual Size:
${ROOTFS_SIZE}, BTRFS Minimum Size: ${MIN_BTRFS_SIZE})"
fi
}
-IMAGE_C
