On Wed, 15 Apr 2020 at 18:56:09 -0600, Aaron Bieber wrote:
> On Mon, 06 Apr 2020 at 17:15:17 -0600, Aaron Bieber wrote:
> > Round two! This time with espie@'s bsd.port.mk fix:
> > https://marc.info/?l=openbsd-ports-cvs&m=158618354824687&w=2
> >
> > This resolves the issue with the conversion of [A-Z] letters to ![a-z] in
> > the
> > package path.
> >
> > With this diff I can generate ports for the following without issue:
> >
> > github.com/jrick/domain
> > github.com/jrick/ss
> > github.com/junegunn/fzf
> > github.com/qbit/gavin
> > golang.zx2c4.com/wireguard
> > humungus.tedunangst.com/r/honk
> > suah.dev/ogvt
> >
> > There are still some issues with things like github.com/restic/restic and
> > github.com/gohugoio/hugo . For some reason the build looks for some files
> > that
> > 'go mod graph' doesn't list.
> >
> > For most go apps that follow the module guidelines:
> > https://github.com/golang/go/wiki/Modules
> >
> > This method of building things should work pretty well. One will have to go
> > in
> > and add a "do-install" to grab other files like man pages, etc.
> >
>
> OK, this fixes some issues with determining the latest version of a given
> port.
>
> With this, I can build a port for restic! (it still needs
> 'ALL_TARGET=github.com/restic/restic/...' set manually)
>
> hugo still results in some missing mod/zip files.
>
Round... 10?
Changes:
- Handle determine latest version in a manner that is more inline with how Go
does it.
- Error out when we are asked to gen a port for an incompatible module.
- Detect licenses from pkg.go.dev.
- Update ALL_TARGET to include "/cmd/..." when a "cmd" directory is present in
a module.
diff --git a/infrastructure/bin/portgen b/infrastructure/bin/portgen
index ad5ab17f3cf..b7316d42b64 100755
--- a/infrastructure/bin/portgen
+++ b/infrastructure/bin/portgen
@@ -32,6 +32,7 @@ use lib ( "$portdir/infrastructure/lib",
"$FindBin::Bin/../lib" );
use OpenBSD::PortGen::Port::CPAN;
use OpenBSD::PortGen::Port::PyPI;
use OpenBSD::PortGen::Port::Ruby;
+use OpenBSD::PortGen::Port::Go;
my ( $type, $module ) = @ARGV;
@@ -44,6 +45,8 @@ if ( $type eq 'p5' ) {
$o = OpenBSD::PortGen::Port::PyPI->new();
} elsif ( $type eq 'ruby' ) {
$o = OpenBSD::PortGen::Port::Ruby->new();
+} elsif ( $type eq 'go' ) {
+ $o = OpenBSD::PortGen::Port::Go->new();
} else {
die "unknown module type\n";
}
diff --git a/infrastructure/lib/OpenBSD/PortGen/Port/Go.pm
b/infrastructure/lib/OpenBSD/PortGen/Port/Go.pm
new file mode 100644
index 00000000000..34668ea6fa7
--- /dev/null
+++ b/infrastructure/lib/OpenBSD/PortGen/Port/Go.pm
@@ -0,0 +1,288 @@
+# $OpenBSD: Go.pm,v 1.16 2019/05/16 16:01:10 afresh1 Exp $
+#
+# Copyright (c) 2019 Aaron Bieber <abie...@openbsd.org>
+#
+# Permission to use, copy, modify, and distribute this software for any
+# purpose with or without fee is hereby granted, provided that the above
+# copyright notice and this permission notice appear in all copies.
+#
+# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+
+package OpenBSD::PortGen::Port::Go;
+
+use 5.028;
+use utf8;
+use warnings;
+use strict;
+use warnings qw(FATAL utf8); # fatalize encoding glitches
+use open qw(:std :encoding(UTF-8)); # undeclared streams in UTF-8
+use OpenBSD::PackageName;
+use OpenBSD::PortGen::Utils qw( fetch );
+
+use parent 'OpenBSD::PortGen::Port';
+
+use Carp;
+use Cwd;
+use File::Temp qw/ tempdir /;
+use Data::Dumper;
+
+use OpenBSD::PortGen::Dependency;
+
+my $license_map = {
+ 'BSD-2-Clause' => 'BSD',
+ 'BSD-3-Clause' => 'BSD3',
+};
+
+sub ecosystem_prefix
+{
+ my $self = shift;
+ return '';
+}
+
+sub base_url
+{
+ my $self = shift;
+ return 'https://proxy.golang.org/';
+}
+
+sub _go_lic_info
+{
+ my ( $self, $module ) = @_;
+ my $html = fetch("https://pkg.go.dev/mod/"; . $module);
+ my $license = "unknown";
+ if ($html =~ m/<a.+tab=licenses.+>(.+)<\/a>/) {
+ $license = $1;
+ $license = $license_map->{$license} || $license;
+ }
+ return $license;
+}
+
+sub _go_determine_name
+{
+ # Some modules end in "v1" or "v2", if we find one of these, we need
+ # to set PKGNAME to something up a level
+ my ( $self, $module ) = @_;
+ my $json = {};
+
+ $json = $self->get_json( $module . '/@latest' );
+
+ if ($json->{Version} =~ m/incompatible/) {
+ my $msg = "${module} $json->{Version} is incompatible with Go
modules.";
+ warn "$msg\n";
+ croak $msg;
+ }
+
+ if ($module =~ m/v\d$/) {
+ $json->{Name} = ( split '/', $module )[-2];
+ } else {
+ $json->{Name} = ( split '/', $module )[-1];
+ }
+
+ $json->{Module} = $module;
+
+ return $json;
+}
+
+sub get_dist_info
+{
+ my ( $self, $module ) = @_;
+
+ my $json = $self->_go_determine_name($module);
+
+ my %mods;
+ for ( $self->_go_mod_graph($json) ) {
+ my ($direct, $ephemeral) = @{$_};
+
+ for my $d ( $direct, $ephemeral ) {
+ next unless $d->{Version};
+ $mods{ $d->{Module} }{ $d->{Version} } ||= $d;
+ }
+ }
+
+ # Filter dependencies to the one with the highest version
+ foreach my $mod ( sort keys %mods ) {
+ # Sort semver numerically then the rest alphanumeric.
+ # This is overkill for sorting, but I already wrote it
+ #
+ # XXX this chokes on the +incompatible likes in some modules
+ my @versions =
+ map { $_->[0] }
+ sort {
+ $a->[1] <=> $b->[1]
+ || $a->[2] <=> $b->[2]
+ || $a->[3] <=> $b->[3]
+ || $a->[4] cmp $b->[4]
+ }
+ map { $_->[1] =~ s/^v//; $_ }
+ map { [ $_, split /[\.-]/, $_, 4 ] }
+ keys %{ $mods{$mod} };
+
+ push @{ $json->{Dist} }, $mods{$mod}{ $versions[-1] };
+ push @{ $json->{Mods} }, map { $mods{$mod}{$_} } @versions;
+ }
+
+ $json->{License} = $self->_go_lic_info($module);
+
+ return $json;
+}
+
+sub _go_mod_graph
+{
+ my ($self, $json) = @_;
+ my $dir = tempdir(CLEANUP => 0);
+
+ my $mod = $self->get("$json->{Module}/\@v/$json->{Version}.mod");
+ my ($module) = $mod =~ /\bmodule\s+(.*)\n/;
+ $module =~ s/\s+$//;
+ unless ( $json->{Module} eq $module ) {
+ my $msg = "Module $json->{Module} doesn't match $module";
+ warn "$msg\n";
+ croak $msg;
+ }
+
+ {
+ open my $fh, '>', $dir . "/go.mod" or die $!;
+ print $fh $mod;
+ close $fh;
+ }
+
+ my $old_cwd = getcwd();
+ chdir $dir or die "Unable to chdir '$dir': $!";
+
+ my @mods;
+ {
+ # Outputs: "direct_dep ephemeral_dep"
+ local $ENV{GOPATH} = "$dir/go";
+ open my $fh, '-|', qw< go mod graph >
+ or die "Unable to spawn 'go mod path': $!";
+ @mods = readline $fh;
+ close $fh
+ or die "Error closing pipe to 'go mod path': $!";
+ }
+
+ chdir $old_cwd or die "Unable to chdir '$old_cwd': $!";
+
+ chomp @mods;
+
+ # parse the graph into pairs of hashrefs
+ return map { [
+ map {
+ my ($m, $v) = split /@/;
+ { Module => $m, Version => $v };
+ } split /\s/
+ ] } grep { $_ } @mods;
+}
+
+sub get_ver_info
+{
+ my ( $self, $module ) = @_;
+ my $version_list = $self->get( $module . '/@v/list' );
+ my $version = "v0.0.0";
+ my $ret;
+
+ # If list isn't populated, it's likely that upstream doesn't follow
+ # semver, this means we will have to fallback to @latest to get the
+ # version information.
+ if ($version_list eq "") {
+ $ret = $self->get_json( $module . '/@latest' );
+ } else {
+ my @parts = split("\n", $version_list);
+ for my $v ( @parts ) {
+ my $a =
OpenBSD::PackageName::version->from_string($version);
+ my $b = OpenBSD::PackageName::version->from_string($v);
+ if ($a->compare($b)) {
+ $version = $v;
+ }
+ }
+ $ret = { Module => $module, Version => $version };
+ }
+
+ return $ret;
+}
+
+sub name_new_port
+{
+ my ( $self, $di ) = @_;
+
+ my $name = $di->{Name};
+ $name = $self->SUPER::name_new_port($name);
+ $name = "go/$name" unless $name =~ m{/};
+
+ return $name;
+}
+
+sub fill_in_makefile
+{
+ my ( $self, $di, $vi ) = @_;
+
+ $self->set_modules('lang/go');
+ $self->set_comment("todo");
+ $self->set_descr("TODO");
+
+ $self->set_license($di->{License});
+
+ $self->set_other( MODGO_MODNAME => $di->{Module} );
+ $self->set_other( MODGO_VERSION => $di->{Version} );
+ $self->set_distname($di->{Name} . '-${MODGO_VERSION}');
+
+ $self->set_pkgname($di->{PkgName}) if defined $di->{PkgName};
+
+ my @parts = split("-", $di->{Version});
+ if (@parts > 1) {
+ $self->set_pkgname($di->{Name} . "-" . $parts[1])
+ if $parts[1] =~ m/\d{6}/;
+ } else {
+ $parts[0] =~ s/^v//;
+ $self->set_pkgname($di->{Name} . "-" . $parts[0]);
+ }
+
+ my @dist = map { [ $_->{Module}, $_->{Version} ] }
+ @{ $di->{Dist} || [] };
+ my @mods = map { [ $_->{Module}, $_->{Version} ] }
+ @{ $di->{Mods} };
+
+ # Turn the deps into tab separated columns
+ foreach my $s ( \@dist, \@mods ) {
+ next unless @{$s}; # if there aren't any, don't try
+ my ($length) = sort { $b <=> $a } map { length $_->[0] } @$s;
+ my $n = ( 1 + int $length / 8 );
+ @{$s} = map {
+ ( my $l = $_->[0] ) =~ s/\p{Upper}/!\L$&/g;
+ my $tabs = "\t" x ( $n - int( length($l) / 8 ) );
+ "$l$tabs $_->[1]"
+ } @{$s};
+ }
+
+ $self->set_other( MODGO_MODULES => \@dist ) if @dist;
+ $self->set_other( MODGO_MODFILES => \@mods ) if @mods;
+}
+
+sub try_building
+{
+ my $self = shift;
+ $self->make_fake();
+}
+
+sub postextract
+{
+}
+
+sub get_deps
+{
+ my ( $self, $di, $wrksrc ) = @_;
+ my $deps = OpenBSD::PortGen::Dependency->new();
+
+ return $deps->format;
+}
+
+sub get_config_style
+{
+}
+
+1;
diff --git a/infrastructure/templates/Makefile.template
b/infrastructure/templates/Makefile.template
index 6be8b86b3ea..2ebf2d37880 100644
--- a/infrastructure/templates/Makefile.template
+++ b/infrastructure/templates/Makefile.template
@@ -22,7 +22,13 @@ COMMENT = ???
#
#MODPY_EGG_VERSION = ???
+# MODGO_MODNAME should be set to the 'module' specified in the 'go.mod' file.
+#MODGO_MODNAME = github.com/test/app
#
+# Version of port if using lang/go and MODGO_MODULES
+#
+#MODGO_VERSION = 0.1.1
+
# What port/package will be created
#
# DISTNAME should not include suffix (like .tar.gz .tgz .tar.bz2 etc.)
@@ -125,6 +131,15 @@ MASTER_SITES = ???
# If port is python3 only
#MODPY_VERSION = ${MODPY_DEFAULT_VERSION_3}
+#
+# MODGO_ settings for when using lang/go module
+#
+# Get source from proxy.golang.org
+#MODGO_MODULES = modulename version
+# These are needed for dependency resolution. We don't actually need the
+# coresponding code
+#MODGO_MODFILES = modulename version
+
# Dependencies
#BUILD_DEPENDS = ???
#RUN_DEPENDS = ???
diff --git a/lang/go/go.port.mk b/lang/go/go.port.mk
index 983c3990706..e6b135ddb1f 100644
--- a/lang/go/go.port.mk
+++ b/lang/go/go.port.mk
@@ -4,6 +4,13 @@ ONLY_FOR_ARCHS ?= ${GO_ARCHS}
MODGO_BUILDDEP ?= Yes
+MODGO_DIST_SUBDIR ?= go_modules
+
+MASTER_SITE_ATHENS = https://proxy.golang.org/
+
+MODGO_MASTER_SITESN = 9
+MASTER_SITES${MODGO_MASTER_SITESN} ?= ${MASTER_SITE_ATHENS}
+
MODGO_RUN_DEPENDS = lang/go
MODGO_BUILD_DEPENDS = lang/go
@@ -33,17 +40,12 @@ MODGO_TYPE ?= bin
MODGO_WORKSPACE ?= ${WRKDIR}/go
MODGO_GOCACHE ?= ${WRKDIR}/go-cache
MODGO_GOPATH ?= ${MODGO_WORKSPACE}:${MODGO_PACKAGE_PATH}
-MAKE_ENV += GOCACHE="${MODGO_GOCACHE}" \
- GOPATH="${MODGO_GOPATH}" \
- GO111MODULE=off
# We cannot assume that the maching running the built code will have SSE,
# even though the machine building the package has SSE. As such, we need
# to explicitly disable SSE on i386 builds.
MAKE_ENV += GO386=387
-# Ports are not allowed to fetch from the network at build time; point
-# GOPROXY at an unreachable host so that failures are also visible to
-# developers who don't have PORTS_PRIVSEP and a "deny .. _pbuild" PF rule.
-MAKE_ENV += GOPROXY=invalid://ports.should.not.fetch.at.buildtime/
+MAKE_ENV += GOCACHE="${MODGO_GOCACHE}"
+
MODGO_CMD ?= ${SETENV} ${MAKE_ENV} go
MODGO_BUILD_CMD = ${MODGO_CMD} install ${MODGO_FLAGS}
MODGO_TEST_CMD = ${MODGO_CMD} test ${MODGO_FLAGS} ${MODGO_TEST_FLAGS}
@@ -54,19 +56,38 @@ MODGO_BUILD_CMD += -ldflags="${MODGO_LDFLAGS}"
MODGO_TEST_CMD += -ldflags="${MODGO_LDFLAGS}"
.endif
-.if defined(GH_ACCOUNT) && defined(GH_PROJECT)
-ALL_TARGET ?= github.com/${GH_ACCOUNT}/${GH_PROJECT}
+.if defined(MODGO_MODNAME)
+EXTRACT_SUFX ?= .zip
+PKGNAME ?= ${DISTNAME:S/-v/-/}
+ALL_TARGET ?= ${MODGO_MODNAME}
+DISTFILES =
${DISTNAME}${EXTRACT_SUFX}{${MODGO_VERSION}${EXTRACT_SUFX}}
+MASTER_SITES ?= ${MASTER_SITE_ATHENS}${MODGO_MODNAME}/@v/
+. for _modpath _modver in ${MODGO_MODULES}
+SUPDISTFILES +=
${MODGO_DIST_SUBDIR}/${_modpath}/@v/${_modver}.zip{${_modpath}/@v/${_modver}.zip}:${MODGO_MASTER_SITESN}
+. endfor
+. for _modpath _modver in ${MODGO_MODFILES}
+SUPDISTFILES +=
${MODGO_DIST_SUBDIR}/${_modpath}/@v/${_modver}.mod{${_modpath}/@v/${_modver}.mod}:${MODGO_MASTER_SITESN}
+. endfor
+MAKE_ENV += GOPROXY=file://${DISTDIR}/${MODGO_DIST_SUBDIR}
+MAKE_ENV += GO111MODULE=on GOPATH="${MODGO_GOPATH}"
+.else
+# ports are not allowed to fetch from the network at build time; point
+# GOPROXY at an unreachable host so that failures are also visible to
+# developers who don't have PORTS_PRIVSEP and a "deny .. _pbuild" PF rule.
+MAKE_ENV += GOPROXY=invalid://ports.should.not.fetch.at.buildtime/
+MAKE_ENV += GO111MODULE=off GOPATH="${MODGO_GOPATH}"
+. if defined(GH_ACCOUNT) && defined(GH_PROJECT)
+ALL_TARGET ?= github.com/${GH_ACCOUNT}/${GH_PROJECT}
+. endif
.endif
-TEST_TARGET ?= ${ALL_TARGET}
-SEPARATE_BUILD ?= Yes
-WRKSRC ?= ${MODGO_WORKSPACE}/src/${ALL_TARGET}
+MODGO_TEST_TARGET ?= cd ${WRKSRC} && ${MODGO_CMD} test ${ALL_TARGET}
-MODGO_SETUP_WORKSPACE = mkdir -p ${WRKSRC:H}; mv ${MODGO_SUBDIR}
${WRKSRC};
+SEPARATE_BUILD ?= Yes
CATEGORIES += lang/go
-MODGO_BUILD_TARGET = ${MODGO_BUILD_CMD} ${ALL_TARGET}
+#MODGO_BUILD_TARGET = ${MODGO_BUILD_CMD}
MODGO_FLAGS += -v -p ${MAKE_JOBS}
.if empty(DEBUG)
@@ -76,6 +97,14 @@ MODGO_LDFLAGS += -s -w
MODGO_FLAGS += -x
.endif
+.if empty(MODGO_MODNAME)
+WRKSRC ?= ${MODGO_WORKSPACE}/src/${ALL_TARGET}
+MODGO_SETUP_WORKSPACE = mkdir -p ${WRKSRC:H}; mv ${MODGO_SUBDIR}
${WRKSRC};
+.else
+WRKSRC ?= ${WRKDIR}/${MODGO_MODNAME}@${MODGO_VERSION}
+MODGO_SETUP_WORKSPACE = ln -sf ${WRKSRC} ${WRKDIR}/${MODGO_MODNAME}
+.endif
+
INSTALL_STRIP =
.if ${MODGO_TYPE:L:Mbin}
MODGO_INSTALL_TARGET = ${INSTALL_PROGRAM_DIR} ${PREFIX}/${MODGO_BINDIR} && \
@@ -99,14 +128,18 @@ MODGO_INSTALL_TARGET += ${INSTALL_DATA_DIR}
${MODGO_PACKAGE_PATH} && \
RUN_DEPENDS += ${MODGO_RUN_DEPENDS}
.endif
-MODGO_TEST_TARGET = ${MODGO_TEST_CMD} ${TEST_TARGET}
-
.if empty(CONFIGURE_STYLE)
MODGO_pre-configure += ${MODGO_SETUP_WORKSPACE}
-. if !target(do-build)
+. if !target(do-install)
do-build:
- ${MODGO_BUILD_TARGET}
+ if [ -d ${WRKSRC}/cmd ]; then \
+ cd ${WRKSRC} && \
+ ${MODGO_BUILD_CMD} ${ALL_TARGET}/cmd/... ; \
+ else \
+ cd ${WRKSRC} && \
+ ${MODGO_BUILD_CMD} ${ALL_TARGET}; \
+ fi;
. endif
. if !target(do-install)
