This is an automated email from the ASF dual-hosted git repository.
ccollins pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/mynewt-newt.git
commit 9be9346655b3fb936f610f923438653b31897d2c
Author: Christopher Collins <ccoll...@apache.org>
AuthorDate: Fri Sep 20 12:17:49 2019 -0500
Run custom commands at build time
This commit adds the ability to run custom commands at build time.
This PR adds the ability to run custom commands at build time.
A package specifies custom commands in its `pkg.yml` file. There are
three types of commands:
1. pre_build_cmds (run before the build)
2. pre_link_cmds (run after compilation, before linking)
3. post_build_cmds (run after the build)
Example (apps/blinky/pkg.yml):
```
pkg.pre_build_cmds:
scripts/pre_build1.sh: 100
scripts/pre_build2.sh: 200
pkg.pre_link_cmds:
scripts/pre_link.sh: 500
pkg.post_build_cmds:
scripts/post_build.sh: 100
```
For each command, the string on the left specifies the command to run.
The number on the right indicates the command's relative ordering.
When newt builds this example, it performs the following sequence:
* scripts/pre_build1.sh
* scripts/pre_build2.sh
* [compile]
* scripts/pre_link.sh
* [link]
* scripts/post_build.sh
If other packages specify custom commands, those commands would also be
executed during the above sequence. For example, if another package
specifies a pre command with an ordering of 150, that command would run
immediately after `pre_build1.sh`. In the case of a tie, the commands
are run in lexicographic order.
All commands are run from the project's base directory. In the above
example, the `scripts` directory is a sibling of `targets`.
See <https://github.com/apache/mynewt-newt/pull/335> for details.
---
go.mod | 1 +
go.sum | 2 +
newt/builder/build.go | 45 +++++++--
newt/builder/buildutil.go | 57 ++++++++++-
newt/builder/extcmd.go | 235 ++++++++++++++++++++++++++++++++++++++++++++
newt/builder/load.go | 13 +--
newt/builder/paths.go | 32 ++++++
newt/builder/targetbuild.go | 42 +++++++-
newt/extcmd/extcmd.go | 98 ++++++++++++++++++
newt/pkg/localpackage.go | 18 ++++
newt/resolve/resolve.go | 29 ++++++
newt/stage/stage.go | 6 +-
util/util.go | 118 ++++++++++++++++++++++
13 files changed, 672 insertions(+), 24 deletions(-)
diff --git a/go.mod b/go.mod
index 5744c4d..e0368d0 100644
--- a/go.mod
+++ b/go.mod
@@ -5,6 +5,7 @@ go 1.12
require (
github.com/NickBall/go-aes-key-wrap v0.0.0-20170929221519-1c3aa3e4dfc5
github.com/apache/mynewt-artifact v0.0.3
+ github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51
github.com/sirupsen/logrus v1.4.2
github.com/spf13/cast v1.3.0
github.com/spf13/cobra v0.0.5
diff --git a/go.sum b/go.sum
index d591003..de1f0bd 100644
--- a/go.sum
+++ b/go.sum
@@ -16,6 +16,8 @@ github.com/davecgh/go-spew v1.1.1/go.mod
h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs
github.com/fsnotify/fsnotify v1.4.7/go.mod
h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
github.com/hashicorp/hcl v1.0.0/go.mod
h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
github.com/inconshreveable/mousetrap v1.0.0/go.mod
h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
+github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51
h1:Z9n2FFNUXsshfwJMBgNA0RU6/i7WVaAegv3PtuIHPMs=
+github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51/go.mod
h1:CzGEWj7cYgsdH8dAjBGEr58BoE7ScuLd+fwFZ44+/x8=
github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod
h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
github.com/magiconair/properties v1.8.0/go.mod
h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
github.com/mitchellh/go-homedir v1.1.0/go.mod
h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
diff --git a/newt/builder/build.go b/newt/builder/build.go
index 1e3c392..d23d44d 100644
--- a/newt/builder/build.go
+++ b/newt/builder/build.go
@@ -83,10 +83,16 @@ func NewBuilder(
}
}
- // Create a pseudo build package for the generated sysinit code.
+ // Create the pseudo build packages.
if _, err := b.addSysinitBpkg(); err != nil {
return nil, err
}
+ if _, err := b.addUserPreBuildBpkg(); err != nil {
+ return nil, err
+ }
+ if _, err := b.addUserPreLinkBpkg(); err != nil {
+ return nil, err
+ }
for api, rpkg := range apiMap {
bpkg := b.PkgMap[rpkg]
@@ -504,6 +510,10 @@ func (b *Builder) PrepBuild() error {
baseCi.Includes = append(baseCi.Includes,
GeneratedIncludeDir(b.targetPkg.rpkg.Lpkg.FullName()))
+ // All packages have access to the user generated header directory.
+ baseCi.Includes = append(baseCi.Includes,
+ UserPreBuildIncludeDir(b.targetPkg.rpkg.Lpkg.Name()))
+
// Let multiplatform libraries know that a Mynewt binary is being build.
baseCi.Cflags = append(baseCi.Cflags, "-DMYNEWT=1")
@@ -518,17 +528,40 @@ func (b *Builder) AddCompilerInfo(info
*toolchain.CompilerInfo) {
b.compilerInfo.AddCompilerInfo(info)
}
-func (b *Builder) addSysinitBpkg() (*BuildPackage, error) {
- lpkg := pkg.NewLocalPackage(b.targetPkg.rpkg.Lpkg.Repo().(*repo.Repo),
- GeneratedBaseDir(b.targetPkg.rpkg.Lpkg.FullName()))
- lpkg.SetName(pkg.ShortName(b.targetPkg.rpkg.Lpkg) + "-sysinit-" +
- b.buildName)
+// addPseudoBpkg creates a dynamic build package and adds it to the builder.
+func (b *Builder) addPseudoBpkg(name string,
+ dir string) (*BuildPackage, error) {
+
+ lpkg := pkg.NewLocalPackage(b.targetPkg.rpkg.Lpkg.Repo().(*repo.Repo),
dir)
+ lpkg.SetName(fmt.Sprintf(
+ "%s-%s", pkg.ShortName(b.targetPkg.rpkg.Lpkg), name))
lpkg.SetType(pkg.PACKAGE_TYPE_GENERATED)
rpkg := resolve.NewResolvePkg(lpkg)
return b.addPackage(rpkg)
}
+// addSysinitBpkg adds the pseudo sysinit build package to the builder.
+func (b *Builder) addSysinitBpkg() (*BuildPackage, error) {
+ name := fmt.Sprintf("%s-%s", "sysinit", b.buildName)
+ dir := GeneratedBaseDir(b.targetPkg.rpkg.Lpkg.FullName())
+ return b.addPseudoBpkg(name, dir)
+}
+
+// addUserPreBuildBpkg adds the pseudo user build package to the builder. The
+// user build package contains inputs emitted by external scripts.
+func (b *Builder) addUserPreBuildBpkg() (*BuildPackage, error) {
+ return b.addPseudoBpkg("user-pre-build",
+ UserPreBuildDir(b.targetPkg.rpkg.Lpkg.FullName()))
+}
+
+// addUserPreLinkBpkg adds the pseudo user build package to the builder. The
+// user build package contains inputs emitted by external scripts.
+func (b *Builder) addUserPreLinkBpkg() (*BuildPackage, error) {
+ return b.addPseudoBpkg("user-pre-link",
+ UserPreLinkDir(b.targetPkg.rpkg.Lpkg.FullName()))
+}
+
// Runs build jobs while any remain. On failure, signals the other workers to
// stop via the stop channel. On error, the error object is signaled via the
// results channel. On successful completion, nil is signaled via the results
diff --git a/newt/builder/buildutil.go b/newt/builder/buildutil.go
index e06bd12..e330f36 100644
--- a/newt/builder/buildutil.go
+++ b/newt/builder/buildutil.go
@@ -21,6 +21,7 @@ package builder
import (
"bytes"
+ "fmt"
"sort"
"strconv"
"strings"
@@ -217,6 +218,42 @@ func SlotEnvVars(bspPkg *pkg.BspPackage,
return env, nil
}
+type UserEnvParams struct {
+ Lpkg *pkg.LocalPackage
+ TargetName string // Short name
+ AppName string
+ BuildName string // "app" or "loader"
+ UserSrcDir string // "" if none
+ UserIncDir string // "" if none
+ WorkDir string
+}
+
+// UserEnvVars calculates the set of environment variables required by external
+// user scripts.
+func UserEnvVars(params UserEnvParams) map[string]string {
+ m := map[string]string{}
+
+ m["MYNEWT_APP_BIN_DIR"] = FileBinDir(
+ params.TargetName, params.BuildName, params.AppName)
+ m["MYNEWT_PKG_BIN_ARCHIVE"] = ArchivePath(
+ params.TargetName, params.BuildName, params.Lpkg.FullName(),
+ params.Lpkg.Type())
+ m["MYNEWT_PKG_BIN_DIR"] = PkgBinDir(
+ params.TargetName, params.BuildName, params.Lpkg.FullName(),
+ params.Lpkg.Type())
+ m["MYNEWT_PKG_NAME"] = params.Lpkg.FullName()
+ m["MYNEWT_USER_WORK_DIR"] = params.WorkDir
+
+ if params.UserSrcDir != "" {
+ m["MYNEWT_USER_SRC_DIR"] = params.UserSrcDir
+ }
+ if params.UserIncDir != "" {
+ m["MYNEWT_USER_INCLUDE_DIR"] = params.UserIncDir
+ }
+
+ return m
+}
+
// EnvVars calculates the full set of environment variables passed to external
// scripts.
func (b *Builder) EnvVars(imageSlot int) (map[string]string, error) {
@@ -238,7 +275,7 @@ func (b *Builder) EnvVars(imageSlot int)
(map[string]string, error) {
imageSlot = -1
}
- slotEnv, err := SlotEnvVars(bspPkg, imageSlot, settings)
+ slotEnv, err := SlotEnvVars(bspPkg, imageSlot)
if err != nil {
return nil, err
}
@@ -252,3 +289,21 @@ func (b *Builder) EnvVars(imageSlot int)
(map[string]string, error) {
return env, nil
}
+
+// EnvVarsToSlice converts an environment variable map into a slice of strings
+// suitable for "shell command" functions defined in `util` (e.g.,
+// util.ShellCommand).
+func EnvVarsToSlice(env map[string]string) []string {
+ keys := make([]string, 0, len(env))
+ for k, _ := range env {
+ keys = append(keys, k)
+ }
+ sort.Strings(keys)
+
+ slice := make([]string, 0, len(env))
+ for _, key := range keys {
+ slice = append(slice, fmt.Sprintf("%s=%s", key, env[key]))
+ }
+
+ return slice
+}
diff --git a/newt/builder/extcmd.go b/newt/builder/extcmd.go
new file mode 100644
index 0000000..624a025
--- /dev/null
+++ b/newt/builder/extcmd.go
@@ -0,0 +1,235 @@
+package builder
+
+import (
+ "io/ioutil"
+ "os"
+ "os/exec"
+
+ "github.com/kballard/go-shellquote"
+ log "github.com/sirupsen/logrus"
+ "mynewt.apache.org/newt/newt/stage"
+ "mynewt.apache.org/newt/util"
+)
+
+// replaceArtifactsIfChanged compares the artifacts just produced (temp
+// directory) to those from the previous build (user bin directory). If they
+// are different, it replaces the old with the new so that they get relinked
+// during this build.
+func replaceArtifactsIfChanged(oldDir string, newDir string) error {
+ eq, err := util.DirsAreEqual(oldDir, newDir)
+ if err != nil {
+ return err
+ }
+
+ if eq {
+ // No changes detected.
+ return nil
+ }
+
+ log.Debugf("changes detected; replacing %s with %s", oldDir, newDir)
+ os.RemoveAll(oldDir)
+ if err := os.Rename(newDir, oldDir); err != nil {
+ return util.ChildNewtError(err)
+ }
+
+ return nil
+}
+
+// createTempUserDirs creates a set of temporary directories for holding build
+// inputs. It returns:
+// * base-dir
+// * src-dir
+// * include-dir
+func createTempUserDirs(label string) (string, string, string, error) {
+ tmpDir, err := ioutil.TempDir("", "mynewt-user-"+label)
+ if err != nil {
+ return "", "", "", util.ChildNewtError(err)
+ }
+ log.Debugf("created user %s dir: %s", label, tmpDir)
+
+ tmpSrcDir := UserTempSrcDir(tmpDir)
+ log.Debugf("creating user %s src dir: %s", label, tmpSrcDir)
+ if err := os.MkdirAll(tmpSrcDir, 0755); err != nil {
+ os.RemoveAll(tmpDir)
+ return "", "", "", util.ChildNewtError(err)
+ }
+
+ tmpIncDir := UserTempIncludeDir(tmpDir)
+ log.Debugf("creating user %s include dir: %s", label, tmpIncDir)
+ if err := os.MkdirAll(tmpIncDir, 0755); err != nil {
+ os.RemoveAll(tmpDir)
+ return "", "", "", util.ChildNewtError(err)
+ }
+
+ return tmpDir, tmpSrcDir, tmpIncDir, nil
+}
+
+// envVarsForCmd calculates the set of environment variables to export for the
+// specified external command.
+func (t *TargetBuilder) envVarsForCmd(sf stage.StageFunc, userSrcDir string,
+ userIncDir string, workDir string) (map[string]string, error) {
+
+ // Determine whether the owning package is part of the loader or the
app.
+ slot := 0
+ buildName := "app"
+
+ if t.LoaderBuilder != nil {
+ rpkg := t.res.LpkgRpkgMap[sf.Pkg]
+ if rpkg == nil {
+ return nil, util.FmtNewtError(
+ "resolution missing expected package: %s",
sf.Pkg.FullName())
+ }
+
+ if t.LoaderBuilder.PkgMap[rpkg] != nil {
+ buildName = "loader"
+ } else {
+ slot = 1
+ }
+ }
+
+ env, err := t.AppBuilder.EnvVars(slot)
+ if err != nil {
+ return nil, err
+ }
+
+ p := UserEnvParams{
+ Lpkg: sf.Pkg,
+ TargetName: t.target.FullName(),
+ AppName: t.appPkg.FullName(),
+ BuildName: buildName,
+ UserSrcDir: userSrcDir,
+ UserIncDir: userIncDir,
+ WorkDir: workDir,
+ }
+ uenv := UserEnvVars(p)
+
+ for k, v := range uenv {
+ env[k] = v
+ }
+
+ return env, nil
+}
+
+// execExtCmds executes a set of user scripts.
+func (t *TargetBuilder) execExtCmds(sf stage.StageFunc, userSrcDir string,
+ userIncDir string, workDir string) error {
+
+ env, err := t.envVarsForCmd(sf, userSrcDir, userIncDir, workDir)
+ if err != nil {
+ return err
+ }
+
+ envs := EnvVarsToSlice(env)
+ toks, err := shellquote.Split(sf.Name)
+ if err != nil {
+ return util.FmtNewtError(
+ "invalid command string: \"%s\": %s", sf.Name,
err.Error())
+ }
+
+ // If the command is in the user's PATH, expand it to its real location.
+ cmd, err := exec.LookPath(toks[0])
+ if err == nil {
+ toks[0] = cmd
+ }
+
+ util.StatusMessage(util.VERBOSITY_DEFAULT, "Executing %s\n", sf.Name)
+ if err := util.ShellInteractiveCommand(toks, envs, true); err != nil {
+ return err
+ }
+
+ return nil
+}
+
+// execPreBuildCmds runs the target's set of pre-build user commands. It is an
+// error if any command fails (exits with a nonzero status).
+func (t *TargetBuilder) execPreBuildCmds(workDir string) error {
+ // Create temporary directories where scripts can put build inputs.
+ tmpDir, tmpSrcDir, tmpIncDir, err := createTempUserDirs("pre-build")
+ if err != nil {
+ return err
+ }
+ defer func() {
+ log.Debugf("removing user pre-build dir: %s", tmpDir)
+ os.RemoveAll(tmpDir)
+ }()
+
+ for _, sf := range t.res.PreBuildCmdCfg.StageFuncs {
+ if err := t.execExtCmds(sf, tmpSrcDir, tmpIncDir, workDir); err
!= nil {
+ return err
+ }
+ }
+
+ srcDir := UserPreBuildSrcDir(t.target.FullName())
+ if err := replaceArtifactsIfChanged(srcDir, tmpSrcDir); err != nil {
+ return err
+ }
+
+ incDir := UserPreBuildIncludeDir(t.target.FullName())
+ if err := replaceArtifactsIfChanged(incDir, tmpIncDir); err != nil {
+ return err
+ }
+
+ return nil
+}
+
+// execPreLinkCmds runs the target's set of post-build user commands. It is
+// an error if any command fails (exits with a nonzero status).
+func (t *TargetBuilder) execPreLinkCmds(workDir string) error {
+ // Create temporary directories where scripts can put build inputs.
+ tmpDir, tmpSrcDir, _, err := createTempUserDirs("pre-link")
+ if err != nil {
+ return err
+ }
+ defer func() {
+ log.Debugf("removing user pre-link dir: %s", tmpDir)
+ os.RemoveAll(tmpDir)
+ }()
+
+ for _, sf := range t.res.PreLinkCmdCfg.StageFuncs {
+ if err := t.execExtCmds(sf, tmpSrcDir, "", workDir); err != nil
{
+ return err
+ }
+ }
+
+ srcDir := UserPreLinkSrcDir(t.target.FullName())
+ err = replaceArtifactsIfChanged(srcDir, tmpSrcDir)
+ if err != nil {
+ return err
+ }
+
+ return nil
+}
+
+// execPostBuildCmds runs the target's set of post-build user commands. It is
+// an error if any command fails (exits with a nonzero status).
+func (t *TargetBuilder) execPostBuildCmds(workDir string) error {
+ for _, sf := range t.res.PostBuildCmdCfg.StageFuncs {
+ if err := t.execExtCmds(sf, "", "", workDir); err != nil {
+ return err
+ }
+ }
+
+ return nil
+}
+
+// makeUserDir creates a temporary directory where scripts can put build
+// inputs.
+func makeUserDir() (string, error) {
+ tmpDir, err := ioutil.TempDir("", "mynewt-user")
+ if err != nil {
+ return "", util.ChildNewtError(err)
+ }
+ log.Debugf("created user dir: %s", tmpDir)
+
+ return tmpDir, nil
+}
+
+func makeUserWorkDir() (string, error) {
+ tmpDir, err := ioutil.TempDir("", "mynewt-user-work")
+ if err != nil {
+ return "", util.ChildNewtError(err)
+ }
+ log.Debugf("created user work dir: %s", tmpDir)
+
+ return tmpDir, nil
+}
diff --git a/newt/builder/load.go b/newt/builder/load.go
index 940a934..a166b8c 100644
--- a/newt/builder/load.go
+++ b/newt/builder/load.go
@@ -23,7 +23,6 @@ import (
"fmt"
"os"
"os/signal"
- "sort"
"strings"
"syscall"
@@ -126,17 +125,7 @@ func Load(binBasePath string, bspPkg *pkg.BspPackage,
for k, v := range extraEnvSettings {
env[k] = v
}
-
- sortedKeys := make([]string, 0, len(env))
- for k, _ := range env {
- sortedKeys = append(sortedKeys, k)
- }
- sort.Strings(sortedKeys)
-
- envSlice := []string{}
- for _, key := range sortedKeys {
- envSlice = append(envSlice, fmt.Sprintf("%s=%s", key, env[key]))
- }
+ envSlice := EnvVarsToSlice(env)
RunOptionalCheck(bspPkg.OptChkScript, envSlice)
// bspPath, binBasePath are passed in command line for backwards
diff --git a/newt/builder/paths.go b/newt/builder/paths.go
index 2fb078e..cc5c2d3 100644
--- a/newt/builder/paths.go
+++ b/newt/builder/paths.go
@@ -63,6 +63,38 @@ func SysinitArchivePath(targetName string) string {
return GeneratedBinDir(targetName) + "/sysinit.a"
}
+func UserBaseDir(targetName string) string {
+ return BinRoot() + "/" + targetName + "/user"
+}
+
+func UserPreBuildDir(targetName string) string {
+ return UserBaseDir(targetName) + "/pre_build"
+}
+
+func UserPreLinkDir(targetName string) string {
+ return UserBaseDir(targetName) + "/pre_link"
+}
+
+func UserPreBuildSrcDir(targetName string) string {
+ return UserPreBuildDir(targetName) + "/src"
+}
+
+func UserPreBuildIncludeDir(targetName string) string {
+ return UserPreBuildDir(targetName) + "/include"
+}
+
+func UserPreLinkSrcDir(targetName string) string {
+ return UserPreLinkDir(targetName) + "/src"
+}
+
+func UserTempSrcDir(tempDir string) string {
+ return tempDir + "/src"
+}
+
+func UserTempIncludeDir(tempDir string) string {
+ return tempDir + "/include"
+}
+
func PkgSyscfgPath(pkgPath string) string {
return pkgPath + "/" + pkg.SYSCFG_YAML_FILENAME
}
diff --git a/newt/builder/targetbuild.go b/newt/builder/targetbuild.go
index 3b1604d..c5145a1 100644
--- a/newt/builder/targetbuild.go
+++ b/newt/builder/targetbuild.go
@@ -330,7 +330,22 @@ func (t *TargetBuilder) PrepBuild() error {
return err
}
- var err error
+ // Create directories where user scripts can write artifacts to
incorporate
+ // into the build.
+
+ err := os.MkdirAll(UserPreBuildSrcDir(t.target.FullName()), 0755)
+ if err != nil {
+ return util.NewNewtError(err.Error())
+ }
+ err = os.MkdirAll(UserPreBuildIncludeDir(t.target.FullName()), 0755)
+ if err != nil {
+ return util.NewNewtError(err.Error())
+ }
+ err = os.MkdirAll(UserPreLinkSrcDir(t.target.FullName()), 0755)
+ if err != nil {
+ return util.NewNewtError(err.Error())
+ }
+
if t.res.LoaderSet != nil {
t.LoaderBuilder, err = NewBuilder(t, BUILD_NAME_LOADER,
t.res.LoaderSet.Rpkgs, t.res.ApiMap, t.res.Cfg)
@@ -483,7 +498,6 @@ func (t *TargetBuilder) Build() error {
return err
}
- /* Build the Apps */
project.ResetDeps(t.AppList)
if err := t.bspPkg.Reload(t.AppBuilder.cfg.SettingValues()); err != nil
{
@@ -497,6 +511,20 @@ func (t *TargetBuilder) Build() error {
}
}
+ workDir, err := makeUserWorkDir()
+ if err != nil {
+ return err
+ }
+ defer func() {
+ log.Debugf("removing user work dir: %s", workDir)
+ os.RemoveAll(workDir)
+ }()
+
+ // Execute the set of pre-build user scripts.
+ if err := t.execPreBuildCmds(workDir); err != nil {
+ return err
+ }
+
if err := t.AppBuilder.Build(); err != nil {
return err
}
@@ -511,11 +539,21 @@ func (t *TargetBuilder) Build() error {
linkerScripts = t.bspPkg.Part2LinkerScripts
}
+ // Execute the set of pre-link user scripts.
+ if err := t.execPreLinkCmds(workDir); err != nil {
+ return err
+ }
+
/* Link the app. */
if err := t.AppBuilder.Link(linkerScripts); err != nil {
return err
}
+ // Execute the set of post-build user scripts.
+ if err := t.execPostBuildCmds(workDir); err != nil {
+ return err
+ }
+
return nil
}
diff --git a/newt/extcmd/extcmd.go b/newt/extcmd/extcmd.go
new file mode 100644
index 0000000..68cfc65
--- /dev/null
+++ b/newt/extcmd/extcmd.go
@@ -0,0 +1,98 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package extcmd
+
+import (
+ "fmt"
+
+ "mynewt.apache.org/newt/newt/pkg"
+ "mynewt.apache.org/newt/newt/stage"
+ "mynewt.apache.org/newt/newt/syscfg"
+)
+
+// ExtCmdCfg represents an ordered list of external commands that get run
+// during the build (aka "user scripts").
+type ExtCmdCfg struct {
+ // Used in diagnostic messages only (example: "pre_build_cmds").
+ Name string
+
+ // Sorted in call order (stage-num,command-string).
+ StageFuncs []stage.StageFunc
+
+ // Strings describing errors encountered while parsing the extcmd
config.
+ InvalidSettings []string
+}
+
+// GetMapFn retrieves a set of ordered external commands from a package. For
+// example, one GetMapFn instance might retrieve a package's pre-build
+// commands.
+type GetMapFn func(
+ lpkg *pkg.LocalPackage, settings map[string]string) map[string]string
+
+func (ecfg *ExtCmdCfg) readOnePkg(lpkg *pkg.LocalPackage, cfg *syscfg.Cfg,
+ getMapCb GetMapFn) {
+
+ settings := cfg.AllSettingsForLpkg(lpkg)
+ cmdMap := getMapCb(lpkg, settings)
+
+ for name, stageStr := range cmdMap {
+ sf, err := stage.NewStageFunc(name, stageStr, lpkg, cfg)
+ if err != nil {
+ text := fmt.Sprintf("%s: %s", lpkg.FullName(),
err.Error())
+ ecfg.InvalidSettings = append(ecfg.InvalidSettings,
text)
+ } else {
+ ecfg.StageFuncs = append(ecfg.StageFuncs, sf)
+ }
+ }
+}
+
+// Read constructs an external command configuration from a full set of
+// packages.
+func Read(name string, lpkgs []*pkg.LocalPackage, cfg *syscfg.Cfg,
+ getMapCb GetMapFn) ExtCmdCfg {
+
+ ecfg := ExtCmdCfg{
+ Name: name,
+ }
+
+ for _, lpkg := range lpkgs {
+ ecfg.readOnePkg(lpkg, cfg, getMapCb)
+ }
+
+ stage.SortStageFuncs(ecfg.StageFuncs, ecfg.Name)
+
+ return ecfg
+}
+
+// If any errors were encountered while parsing extcmd definitions, this
+// function returns a string indicating the errors. If no errors were
+// encountered, "" is returned.
+func (ecfg *ExtCmdCfg) ErrorText() string {
+ str := ""
+
+ if len(ecfg.InvalidSettings) > 0 {
+ str += fmt.Sprintf("Invalid %s definitions detected:",
ecfg.Name)
+ for _, e := range ecfg.InvalidSettings {
+ str += "\n " + e
+ }
+ }
+
+ return str
+}
diff --git a/newt/pkg/localpackage.go b/newt/pkg/localpackage.go
index fcd2e68..d1f0088 100644
--- a/newt/pkg/localpackage.go
+++ b/newt/pkg/localpackage.go
@@ -377,6 +377,24 @@ func (pkg *LocalPackage) DownFuncs(
return pkg.PkgY.GetValStringMapString("pkg.down", settings)
}
+func (pkg *LocalPackage) PreBuildCmds(
+ settings map[string]string) map[string]string {
+
+ return pkg.PkgY.GetValStringMapString("pkg.pre_build_cmds", settings)
+}
+
+func (pkg *LocalPackage) PreLinkCmds(
+ settings map[string]string) map[string]string {
+
+ return pkg.PkgY.GetValStringMapString("pkg.pre_link_cmds", settings)
+}
+
+func (pkg *LocalPackage) PostBuildCmds(
+ settings map[string]string) map[string]string {
+
+ return pkg.PkgY.GetValStringMapString("pkg.post_build_cmds", settings)
+}
+
func (pkg *LocalPackage) InjectedSettings() map[string]string {
return pkg.injectedSettings
}
diff --git a/newt/resolve/resolve.go b/newt/resolve/resolve.go
index 07fa1de..0cb6c98 100644
--- a/newt/resolve/resolve.go
+++ b/newt/resolve/resolve.go
@@ -26,6 +26,7 @@ import (
log "github.com/sirupsen/logrus"
+ "mynewt.apache.org/newt/newt/extcmd"
"mynewt.apache.org/newt/newt/flashmap"
"mynewt.apache.org/newt/newt/logcfg"
"mynewt.apache.org/newt/newt/newtutil"
@@ -67,6 +68,9 @@ type Resolver struct {
lcfg logcfg.LCfg
sysinitCfg sysinit.SysinitCfg
sysdownCfg sysdown.SysdownCfg
+ preBuildCmdCfg extcmd.ExtCmdCfg
+ preLinkCmdCfg extcmd.ExtCmdCfg
+ postBuildCmdCfg extcmd.ExtCmdCfg
// [api-name][api-supplier]
apiConflicts map[string]map[*ResolvePackage]struct{}
@@ -119,6 +123,9 @@ type Resolution struct {
LCfg logcfg.LCfg
SysinitCfg sysinit.SysinitCfg
SysdownCfg sysdown.SysdownCfg
+ PreBuildCmdCfg extcmd.ExtCmdCfg
+ PreLinkCmdCfg extcmd.ExtCmdCfg
+ PostBuildCmdCfg extcmd.ExtCmdCfg
ApiMap map[string]*ResolvePackage
UnsatisfiedApis map[string][]*ResolvePackage
ApiConflicts []ApiConflict
@@ -987,6 +994,22 @@ func (r *Resolver) resolveDepsAndCfg() error {
r.sysinitCfg = sysinit.Read(lpkgs, &r.cfg)
r.sysdownCfg = sysdown.Read(lpkgs, &r.cfg)
+ r.preBuildCmdCfg = extcmd.Read("pre_build_cmds", lpkgs, &r.cfg,
+ func(lpkg *pkg.LocalPackage,
+ settings map[string]string) map[string]string {
+ return lpkg.PreBuildCmds(settings)
+ })
+ r.preLinkCmdCfg = extcmd.Read("pre_link_cmds", lpkgs, &r.cfg,
+ func(lpkg *pkg.LocalPackage,
+ settings map[string]string) map[string]string {
+ return lpkg.PreLinkCmds(settings)
+ })
+ r.postBuildCmdCfg = extcmd.Read("post_build_cmds", lpkgs, &r.cfg,
+ func(lpkg *pkg.LocalPackage,
+ settings map[string]string) map[string]string {
+ return lpkg.PostBuildCmds(settings)
+ })
+
// Log the final syscfg.
r.cfg.Log()
@@ -1085,6 +1108,9 @@ func ResolveFull(
res.LCfg = r.lcfg
res.SysinitCfg = r.sysinitCfg
res.SysdownCfg = r.sysdownCfg
+ res.PreBuildCmdCfg = r.preBuildCmdCfg
+ res.PreLinkCmdCfg = r.preLinkCmdCfg
+ res.PostBuildCmdCfg = r.postBuildCmdCfg
// Determine which package satisfies each API and which APIs are
// unsatisfied.
@@ -1203,6 +1229,9 @@ func (res *Resolution) ErrorText() string {
str += res.LCfg.ErrorText()
str += res.SysinitCfg.ErrorText()
str += res.SysdownCfg.ErrorText()
+ str += res.PreBuildCmdCfg.ErrorText()
+ str += res.PreLinkCmdCfg.ErrorText()
+ str += res.PostBuildCmdCfg.ErrorText()
str = strings.TrimSpace(str)
if str != "" {
diff --git a/newt/stage/stage.go b/newt/stage/stage.go
index 7ad4e90..dce33c3 100644
--- a/newt/stage/stage.go
+++ b/newt/stage/stage.go
@@ -59,9 +59,9 @@ func NewStageFunc(name string, textVal string,
// Ensure setting resolves to an integer.
if _, err := vs.IntVal(); err != nil {
- return StageFunc{}, util.FmtNewtError("Invalid stage setting:
%s=%s; "+
- "value does not resolve to an integer",
- name, textVal)
+ return StageFunc{}, util.FmtNewtError("Invalid setting: \"%s:
%s\"; "+
+ "value \"%s\" does not resolve to an integer",
+ name, textVal, textVal)
}
sf := StageFunc{
diff --git a/util/util.go b/util/util.go
index a008a2e..aa46495 100644
--- a/util/util.go
+++ b/util/util.go
@@ -786,3 +786,121 @@ func MarshalJSONStringer(sr fmt.Stringer) ([]byte, error)
{
return j, nil
}
+
+// readDirRecursive recursively reads the contents of a directory. It returns
+// [dir-paths],[file-paths]. All returned strings are relative to the provided
+// base directory.
+func readDirRecursive(path string) ([]string, []string, error) {
+ var dirs []string
+ var files []string
+
+ var iter func(crumbs string) error
+ iter = func(crumbs string) error {
+ var crumbsPath string
+ if crumbs != "" {
+ crumbsPath = "/" + crumbs
+ }
+
+ f, err := os.Open(path + crumbsPath)
+ if err != nil {
+ return ChildNewtError(err)
+ }
+ defer f.Close()
+
+ infos, err := f.Readdir(-1)
+ if err != nil {
+ return ChildNewtError(err)
+ }
+
+ for _, info := range infos {
+ name := fmt.Sprintf("%s/%s", crumbs, info.Name())
+
+ if info.IsDir() {
+ dirs = append(dirs, name)
+ if err := iter(name); err != nil {
+ return err
+ }
+ } else {
+ files = append(files, name)
+ }
+ }
+
+ return nil
+ }
+
+ if err := iter(""); err != nil {
+ return nil, nil, err
+ }
+
+ return dirs, files, nil
+}
+
+// DirsAreEqual compares the contents of two directories. Directories are
+// equal if 1) their subdirectory structures are identical, and 2) they contain
+// the exact same set of files (same names and contents).
+func DirsAreEqual(dira string, dirb string) (bool, error) {
+ dirsa, filesa, err := readDirRecursive(dira)
+ if err != nil {
+ return false, err
+ }
+
+ dirsb, filesb, err := readDirRecursive(dirb)
+ if err != nil {
+ return false, err
+ }
+
+ if len(dirsa) != len(dirsb) || len(filesa) != len(filesb) {
+ return false, nil
+ }
+
+ // Returns the intersection of two sets of strings.
+ intersection := func(a []string, b []string) map[string]struct{} {
+ ma := make(map[string]struct{}, len(a))
+ for _, p := range a {
+ ma[p] = struct{}{}
+ }
+
+ isect := map[string]struct{}{}
+ for _, p := range b {
+ if _, ok := ma[p]; ok {
+ isect[p] = struct{}{}
+ }
+ }
+
+ return isect
+ }
+
+ // If the intersection lengths are equal, both directories have the same
+ // structure.
+
+ isectDirs := intersection(dirsa, dirsb)
+ if len(isectDirs) != len(dirsa) {
+ return false, nil
+ }
+
+ isectFiles := intersection(filesa, filesb)
+ if len(isectFiles) != len(filesa) {
+ return false, nil
+ }
+
+ // Finally, compare the contents of files in each directory.
+ for _, p := range filesa {
+ patha := fmt.Sprintf("%s/%s", dira, p)
+ bytesa, err := ioutil.ReadFile(patha)
+ if err != nil {
+ return false, ChildNewtError(err)
+ }
+
+ pathb := fmt.Sprintf("%s/%s", dirb, p)
+ unchanged, err := FileContains(bytesa, pathb)
+ if err != nil {
+ return false, err
+ }
+
+ if !unchanged {
+ return false, nil
+ }
+ }
+
+ return true, nil
+}
