This is an automated email from the ASF dual-hosted git repository. msciabarra pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/openwhisk-runtime-go.git
The following commit(s) were added to refs/heads/master by this push: new 55229db Symlink in zips (#148) 55229db is described below commit 55229db17afd0ce21912598d3178afc668349aaf Author: Michele Sciabarra <30654959+sciabarra...@users.noreply.github.com> AuthorDate: Sat Jul 10 13:50:06 2021 +0200 Symlink in zips (#148) * implemented symlinks in zips * bug fixes Co-authored-by: Michele Sciabarra <mich...@sciabarra.com> Co-authored-by: Michele Sciabarra <git...@sciabarra.com> --- .gitignore | 5 +++ CHANGES.md | 3 +- openwhisk/_test/build.sh | 2 +- openwhisk/util_test.go | 12 +++++++ openwhisk/version.go | 2 +- openwhisk/zip.go | 84 ++++++++++++++++++++++++++++++++++++++---------- openwhisk/zip_test.go | 21 ++++++++++++ 7 files changed, 109 insertions(+), 20 deletions(-) diff --git a/.gitignore b/.gitignore index a06a05b..361c9ec 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,9 @@ # Mac .DS_Store +# goenv +.go-version + # Gradle .gradle/ .gogradle/ @@ -27,10 +30,12 @@ openwhisk/_test/*.zip openwhisk/_test/*.jar openwhisk/_test/compile/ openwhisk/_test/output/ +openwhisk/_test/venv/ openwhisk/action/ openwhisk/compile/ openwhisk/debug.test *.pyc +*.env # Eclipse tests/bin/ diff --git a/CHANGES.md b/CHANGES.md index 44a7c61..6d83c4d 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -17,7 +17,8 @@ # --> -# next release +# 1.17.1 +- support for zipping and unzipping symbolic links (required to support virtualenvs) - go 1.15 runtime upgraded to 1.15.13 # 1.17.0 diff --git a/openwhisk/_test/build.sh b/openwhisk/_test/build.sh index fe17a65..769c58e 100755 --- a/openwhisk/_test/build.sh +++ b/openwhisk/_test/build.sh @@ -60,4 +60,4 @@ zip -q -r exec.zip exec etc dir echo exec/env >helloack/exec.env zip -j helloack.zip helloack/* - +python3 -m venv venv diff --git a/openwhisk/util_test.go b/openwhisk/util_test.go index 586520a..fc0d810 100644 --- a/openwhisk/util_test.go +++ b/openwhisk/util_test.go @@ -138,6 +138,7 @@ func dump(file *os.File) { os.Remove(file.Name()) } +// printing output only if no errors func sys(cli string, args ...string) { os.Chmod(cli, 0755) cmd := exec.Command(cli, args...) @@ -149,6 +150,17 @@ func sys(cli string, args ...string) { } } +// version printing output also when errors +func sys2(cli string, args ...string) { + os.Chmod(cli, 0755) + cmd := exec.Command(cli, args...) + out, err := cmd.CombinedOutput() + fmt.Print(string(out)) + if err != nil { + log.Print(err) + } +} + func exists(dir, filename string) error { path := fmt.Sprintf("%s/%d/%s", dir, highestDir(dir), filename) _, err := os.Stat(path) diff --git a/openwhisk/version.go b/openwhisk/version.go index fd310fe..d8ede5b 100644 --- a/openwhisk/version.go +++ b/openwhisk/version.go @@ -17,4 +17,4 @@ package openwhisk // Version number - internal -var Version = "1.16.0" +var Version = "1.17.1" diff --git a/openwhisk/zip.go b/openwhisk/zip.go index fd4b994..f327a53 100644 --- a/openwhisk/zip.go +++ b/openwhisk/zip.go @@ -60,24 +60,42 @@ func Unzip(src []byte, dest string) error { os.MkdirAll(dest, 0755) // Closure to address file descriptors issue with all the deferred .Close() methods extractAndWriteFile := func(f *zip.File) error { + + path := filepath.Join(dest, f.Name) + isLink := f.FileInfo().Mode()&os.ModeSymlink == os.ModeSymlink + + // dir + if f.FileInfo().IsDir() && !isLink { + return os.MkdirAll(path, f.Mode()) + } + + // open file rc, err := f.Open() - defer rc.Close() if err != nil { return err } - path := filepath.Join(dest, f.Name) - if f.FileInfo().IsDir() { - return os.MkdirAll(path, 0755) + defer rc.Close() + + // link + if isLink { + buf, err := ioutil.ReadAll(rc) + if err != nil { + return err + } + return os.Symlink(string(buf), path) } + + // file + // eventually create a missing ddir err = os.MkdirAll(filepath.Dir(path), 0755) if err != nil { return err } file, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, f.Mode()) - defer file.Close() if err != nil { return err } + defer file.Close() _, err = io.Copy(file, rc) return err } @@ -96,24 +114,56 @@ func Zip(dir string) ([]byte, error) { zwr := zip.NewWriter(buf) dir = filepath.Clean(dir) err := filepath.Walk(dir, func(filePath string, info os.FileInfo, err error) error { - if info.IsDir() { + + // trim the relevant part of the path + relPath := strings.TrimPrefix(filePath, dir) + if relPath == "" { return nil } + relPath = relPath[1:] if err != nil { return err } - relPath := strings.TrimPrefix(filePath, dir)[1:] - zipFile, err := zwr.Create(relPath) - if err != nil { - return err - } - fsFile, err := os.Open(filePath) - if err != nil { - return err + + // create a proper entry + isLink := (info.Mode() & os.ModeSymlink) == os.ModeSymlink + header := &zip.FileHeader{ + Name: relPath, + Method: zip.Deflate, } - _, err = io.Copy(zipFile, fsFile) - if err != nil { - return err + if isLink { + header.SetMode(0755 | os.ModeSymlink) + w, err := zwr.CreateHeader(header) + if err != nil { + return err + } + ln, err := os.Readlink(filePath) + if err != nil { + return err + } + w.Write([]byte(ln)) + } else if info.IsDir() { + header.Name = relPath + "/" + header.SetMode(0755) + _, err := zwr.CreateHeader(header) + if err != nil { + return err + } + } else if info.Mode().IsRegular() { + header.SetMode(0755) + w, err := zwr.CreateHeader(header) + if err != nil { + return err + } + fsFile, err := os.Open(filePath) + if err != nil { + return err + } + defer fsFile.Close() + _, err = io.Copy(w, fsFile) + if err != nil { + return err + } } return nil }) diff --git a/openwhisk/zip_test.go b/openwhisk/zip_test.go index f386ef0..c72d5f3 100644 --- a/openwhisk/zip_test.go +++ b/openwhisk/zip_test.go @@ -18,6 +18,7 @@ package openwhisk import ( "fmt" + "io/ioutil" "os" ) @@ -54,3 +55,23 @@ func Example_jar() { // ./action/unzip/exec.jar // <nil> } + +func Example_venv() { + os.RemoveAll("./action/unzip") + os.Mkdir("./action/unzip", 0755) + buf, err := Zip("_test/venv") + fmt.Println(1, err) + err = ioutil.WriteFile("/tmp/appo.zip", buf, 0644) + fmt.Println(2, err) + err = UnzipOrSaveJar(buf, "./action/unzip", "./action/unzip/exec.jar") + sys("bash", "-c", "cd action/unzip/bin && find . -type l -name python && rm ./python") + sys2("bash", "-c", "diff -qr _test/venv action/unzip 2>/dev/null") + fmt.Println(3, err) + // Output: + // 1 <nil> + // 2 <nil> + // ./python + // Only in _test/venv/bin: python + // 3 <nil> + +}