On Fri, Apr 17, 2020 at 05:26:58PM -0600, Aaron Bieber wrote: > On Wed, 15 Apr 2020 at 18:56:09 -0600, Aaron Bieber wrote: > > On Mon, 06 Apr 2020 at 17:15:17 -0600, Aaron Bieber wrote: > > > Round two! This time with espie@'s bsd.port.mk fix: > > > https://marc.info/?l=openbsd-ports-cvs&m=158618354824687&w=2 > > > > > > This resolves the issue with the conversion of [A-Z] letters to ![a-z] in > > > the > > > package path. > > > > > > With this diff I can generate ports for the following without issue: > > > > > > github.com/jrick/domain > > > github.com/jrick/ss > > > github.com/junegunn/fzf > > > github.com/qbit/gavin > > > golang.zx2c4.com/wireguard > > > humungus.tedunangst.com/r/honk > > > suah.dev/ogvt > > > > > > There are still some issues with things like github.com/restic/restic and > > > github.com/gohugoio/hugo . For some reason the build looks for some files > > > that > > > 'go mod graph' doesn't list. > > > > > > For most go apps that follow the module guidelines: > > > https://github.com/golang/go/wiki/Modules > > > > > > This method of building things should work pretty well. One will have to > > > go in > > > and add a "do-install" to grab other files like man pages, etc. > > > > > > > OK, this fixes some issues with determining the latest version of a given > > port. > > > > With this, I can build a port for restic! (it still needs > > 'ALL_TARGET=github.com/restic/restic/...' set manually) > > > > hugo still results in some missing mod/zip files. > > > > Round... 10?
A few super tiny comments inline, only one I would fail the review for :-) > > Changes: > - Handle determine latest version in a manner that is more inline with how Go > does it. > - Error out when we are asked to gen a port for an incompatible module. > - Detect licenses from pkg.go.dev. > - Update ALL_TARGET to include "/cmd/..." when a "cmd" directory is present in > a module. > > diff --git a/infrastructure/bin/portgen b/infrastructure/bin/portgen > index ad5ab17f3cf..b7316d42b64 100755 > --- a/infrastructure/bin/portgen > +++ b/infrastructure/bin/portgen > @@ -32,6 +32,7 @@ use lib ( "$portdir/infrastructure/lib", > "$FindBin::Bin/../lib" ); > use OpenBSD::PortGen::Port::CPAN; > use OpenBSD::PortGen::Port::PyPI; > use OpenBSD::PortGen::Port::Ruby; > +use OpenBSD::PortGen::Port::Go; > > my ( $type, $module ) = @ARGV; > > @@ -44,6 +45,8 @@ if ( $type eq 'p5' ) { > $o = OpenBSD::PortGen::Port::PyPI->new(); > } elsif ( $type eq 'ruby' ) { > $o = OpenBSD::PortGen::Port::Ruby->new(); > +} elsif ( $type eq 'go' ) { > + $o = OpenBSD::PortGen::Port::Go->new(); > } else { > die "unknown module type\n"; > } > diff --git a/infrastructure/lib/OpenBSD/PortGen/Port/Go.pm > b/infrastructure/lib/OpenBSD/PortGen/Port/Go.pm > new file mode 100644 > index 00000000000..34668ea6fa7 > --- /dev/null > +++ b/infrastructure/lib/OpenBSD/PortGen/Port/Go.pm > @@ -0,0 +1,288 @@ > +# $OpenBSD: Go.pm,v 1.16 2019/05/16 16:01:10 afresh1 Exp $ > +# > +# Copyright (c) 2019 Aaron Bieber <abie...@openbsd.org> > +# > +# Permission to use, copy, modify, and distribute this software for any > +# purpose with or without fee is hereby granted, provided that the above > +# copyright notice and this permission notice appear in all copies. > +# > +# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES > +# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF > +# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR > +# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES > +# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN > +# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF > +# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. > + > +package OpenBSD::PortGen::Port::Go; > + > +use 5.028; > +use utf8; > +use warnings; > +use strict; > +use warnings qw(FATAL utf8); # fatalize encoding glitches > +use open qw(:std :encoding(UTF-8)); # undeclared streams in UTF-8 > +use OpenBSD::PackageName; > +use OpenBSD::PortGen::Utils qw( fetch ); > + > +use parent 'OpenBSD::PortGen::Port'; > + > +use Carp; > +use Cwd; > +use File::Temp qw/ tempdir /; > +use Data::Dumper; > + > +use OpenBSD::PortGen::Dependency; > + > +my $license_map = { > + 'BSD-2-Clause' => 'BSD', > + 'BSD-3-Clause' => 'BSD3', > +}; This could just go into "%good_licenses" in OpenBSD::PortGen::License. https://github.com/openbsd/ports/blob/master/infrastructure/lib/OpenBSD/PortGen/License.pm > + > +sub ecosystem_prefix > +{ > + my $self = shift; > + return ''; > +} > + > +sub base_url > +{ > + my $self = shift; > + return 'https://proxy.golang.org/'; > +} > + > +sub _go_lic_info > +{ > + my ( $self, $module ) = @_; > + my $html = fetch("https://pkg.go.dev/mod/" . $module); > + my $license = "unknown"; > + if ($html =~ m/<a.+tab=licenses.+>(.+)<\/a>/) { > + $license = $1; > + $license = $license_map->{$license} || $license; > + } > + return $license; > +} > + > +sub _go_determine_name > +{ > + # Some modules end in "v1" or "v2", if we find one of these, we need > + # to set PKGNAME to something up a level > + my ( $self, $module ) = @_; > + my $json = {}; > + > + $json = $self->get_json( $module . '/@latest' ); totally fine to just do: my $json = $self->get_json( $module . '/@latest' ); > + > + if ($json->{Version} =~ m/incompatible/) { > + my $msg = "${module} $json->{Version} is incompatible with Go > modules."; > + warn "$msg\n"; > + croak $msg; > + } croak should be printing out this $msg, so this would end up with duplicate output. That seems not great. > + > + if ($module =~ m/v\d$/) { > + $json->{Name} = ( split '/', $module )[-2]; > + } else { > + $json->{Name} = ( split '/', $module )[-1]; > + } > + > + $json->{Module} = $module; > + > + return $json; > +} > + > +sub get_dist_info > +{ > + my ( $self, $module ) = @_; > + > + my $json = $self->_go_determine_name($module); > + > + my %mods; > + for ( $self->_go_mod_graph($json) ) { > + my ($direct, $ephemeral) = @{$_}; > + > + for my $d ( $direct, $ephemeral ) { > + next unless $d->{Version}; > + $mods{ $d->{Module} }{ $d->{Version} } ||= $d; > + } > + } > + > + # Filter dependencies to the one with the highest version > + foreach my $mod ( sort keys %mods ) { > + # Sort semver numerically then the rest alphanumeric. > + # This is overkill for sorting, but I already wrote it > + # > + # XXX this chokes on the +incompatible likes in some modules > + my @versions = > + map { $_->[0] } > + sort { > + $a->[1] <=> $b->[1] > + || $a->[2] <=> $b->[2] > + || $a->[3] <=> $b->[3] > + || $a->[4] cmp $b->[4] > + } > + map { $_->[1] =~ s/^v//; $_ } > + map { [ $_, split /[\.-]/, $_, 4 ] } If you add the `+` to this character class along with `\.-` that may solve the choking on +incompatible. Although I'm not 100% sure what those versions look like. > + keys %{ $mods{$mod} }; > + > + push @{ $json->{Dist} }, $mods{$mod}{ $versions[-1] }; > + push @{ $json->{Mods} }, map { $mods{$mod}{$_} } @versions; > + } > + > + $json->{License} = $self->_go_lic_info($module); > + > + return $json; > +} > + > +sub _go_mod_graph > +{ > + my ($self, $json) = @_; > + my $dir = tempdir(CLEANUP => 0); > + > + my $mod = $self->get("$json->{Module}/\@v/$json->{Version}.mod"); > + my ($module) = $mod =~ /\bmodule\s+(.*)\n/; > + $module =~ s/\s+$//; or my ($module) = $mod =~ /\bmodule\s+(.*?)\s/; > + unless ( $json->{Module} eq $module ) { > + my $msg = "Module $json->{Module} doesn't match $module"; > + warn "$msg\n"; > + croak $msg; > + } > + > + { > + open my $fh, '>', $dir . "/go.mod" or die $!; > + print $fh $mod; > + close $fh; > + } > + > + my $old_cwd = getcwd(); > + chdir $dir or die "Unable to chdir '$dir': $!"; > + > + my @mods; > + { > + # Outputs: "direct_dep ephemeral_dep" > + local $ENV{GOPATH} = "$dir/go"; > + open my $fh, '-|', qw< go mod graph > > + or die "Unable to spawn 'go mod path': $!"; > + @mods = readline $fh; > + close $fh > + or die "Error closing pipe to 'go mod path': $!"; > + } > + > + chdir $old_cwd or die "Unable to chdir '$old_cwd': $!"; > + > + chomp @mods; > + > + # parse the graph into pairs of hashrefs > + return map { [ > + map { > + my ($m, $v) = split /@/; > + { Module => $m, Version => $v }; > + } split /\s/ > + ] } grep { $_ } @mods; > +} > + > +sub get_ver_info > +{ > + my ( $self, $module ) = @_; > + my $version_list = $self->get( $module . '/@v/list' ); > + my $version = "v0.0.0"; > + my $ret; > + > + # If list isn't populated, it's likely that upstream doesn't follow > + # semver, this means we will have to fallback to @latest to get the > + # version information. > + if ($version_list eq "") { > + $ret = $self->get_json( $module . '/@latest' ); > + } else { > + my @parts = split("\n", $version_list); > + for my $v ( @parts ) { > + my $a = > OpenBSD::PackageName::version->from_string($version); > + my $b = OpenBSD::PackageName::version->from_string($v); > + if ($a->compare($b)) { > + $version = $v; > + } > + } > + $ret = { Module => $module, Version => $version }; > + } > + > + return $ret; > +} > + > +sub name_new_port > +{ > + my ( $self, $di ) = @_; > + > + my $name = $di->{Name}; > + $name = $self->SUPER::name_new_port($name); > + $name = "go/$name" unless $name =~ m{/}; > + > + return $name; > +} > + > +sub fill_in_makefile > +{ > + my ( $self, $di, $vi ) = @_; > + > + $self->set_modules('lang/go'); > + $self->set_comment("todo"); > + $self->set_descr("TODO"); > + > + $self->set_license($di->{License}); > + > + $self->set_other( MODGO_MODNAME => $di->{Module} ); > + $self->set_other( MODGO_VERSION => $di->{Version} ); > + $self->set_distname($di->{Name} . '-${MODGO_VERSION}'); > + > + $self->set_pkgname($di->{PkgName}) if defined $di->{PkgName}; > + > + my @parts = split("-", $di->{Version}); > + if (@parts > 1) { > + $self->set_pkgname($di->{Name} . "-" . $parts[1]) > + if $parts[1] =~ m/\d{6}/; > + } else { > + $parts[0] =~ s/^v//; > + $self->set_pkgname($di->{Name} . "-" . $parts[0]); > + } > + > + my @dist = map { [ $_->{Module}, $_->{Version} ] } > + @{ $di->{Dist} || [] }; > + my @mods = map { [ $_->{Module}, $_->{Version} ] } > + @{ $di->{Mods} }; > + > + # Turn the deps into tab separated columns > + foreach my $s ( \@dist, \@mods ) { > + next unless @{$s}; # if there aren't any, don't try > + my ($length) = sort { $b <=> $a } map { length $_->[0] } @$s; > + my $n = ( 1 + int $length / 8 ); > + @{$s} = map { > + ( my $l = $_->[0] ) =~ s/\p{Upper}/!\L$&/g; > + my $tabs = "\t" x ( $n - int( length($l) / 8 ) ); > + "$l$tabs $_->[1]" > + } @{$s}; > + } > + > + $self->set_other( MODGO_MODULES => \@dist ) if @dist; > + $self->set_other( MODGO_MODFILES => \@mods ) if @mods; > +} > + > +sub try_building > +{ > + my $self = shift; > + $self->make_fake(); > +} > + > +sub postextract > +{ > +} > + > +sub get_deps > +{ > + my ( $self, $di, $wrksrc ) = @_; > + my $deps = OpenBSD::PortGen::Dependency->new(); > + > + return $deps->format; > +} > + > +sub get_config_style > +{ > +} > + > +1; No idea on the Makefile stuff :-) > diff --git a/infrastructure/templates/Makefile.template > b/infrastructure/templates/Makefile.template > index 6be8b86b3ea..2ebf2d37880 100644 > --- a/infrastructure/templates/Makefile.template > +++ b/infrastructure/templates/Makefile.template > @@ -22,7 +22,13 @@ COMMENT = ??? > # > #MODPY_EGG_VERSION = ??? > > +# MODGO_MODNAME should be set to the 'module' specified in the 'go.mod' file. > +#MODGO_MODNAME = github.com/test/app > # > +# Version of port if using lang/go and MODGO_MODULES > +# > +#MODGO_VERSION = 0.1.1 > + > # What port/package will be created > # > # DISTNAME should not include suffix (like .tar.gz .tgz .tar.bz2 etc.) > @@ -125,6 +131,15 @@ MASTER_SITES = ??? > # If port is python3 only > #MODPY_VERSION = ${MODPY_DEFAULT_VERSION_3} > > +# > +# MODGO_ settings for when using lang/go module > +# > +# Get source from proxy.golang.org > +#MODGO_MODULES = modulename version > +# These are needed for dependency resolution. We don't actually need the > +# coresponding code > +#MODGO_MODFILES = modulename version > + > # Dependencies > #BUILD_DEPENDS = ??? > #RUN_DEPENDS = ??? > diff --git a/lang/go/go.port.mk b/lang/go/go.port.mk > index 983c3990706..e6b135ddb1f 100644 > --- a/lang/go/go.port.mk > +++ b/lang/go/go.port.mk > @@ -4,6 +4,13 @@ ONLY_FOR_ARCHS ?= ${GO_ARCHS} > > MODGO_BUILDDEP ?= Yes > > +MODGO_DIST_SUBDIR ?= go_modules > + > +MASTER_SITE_ATHENS = https://proxy.golang.org/ > + > +MODGO_MASTER_SITESN = 9 > +MASTER_SITES${MODGO_MASTER_SITESN} ?= ${MASTER_SITE_ATHENS} > + > MODGO_RUN_DEPENDS = lang/go > MODGO_BUILD_DEPENDS = lang/go > > @@ -33,17 +40,12 @@ MODGO_TYPE ?= bin > MODGO_WORKSPACE ?= ${WRKDIR}/go > MODGO_GOCACHE ?= ${WRKDIR}/go-cache > MODGO_GOPATH ?= ${MODGO_WORKSPACE}:${MODGO_PACKAGE_PATH} > -MAKE_ENV += GOCACHE="${MODGO_GOCACHE}" \ > - GOPATH="${MODGO_GOPATH}" \ > - GO111MODULE=off > # We cannot assume that the maching running the built code will have SSE, > # even though the machine building the package has SSE. As such, we need > # to explicitly disable SSE on i386 builds. > MAKE_ENV += GO386=387 > -# Ports are not allowed to fetch from the network at build time; point > -# GOPROXY at an unreachable host so that failures are also visible to > -# developers who don't have PORTS_PRIVSEP and a "deny .. _pbuild" PF rule. > -MAKE_ENV += GOPROXY=invalid://ports.should.not.fetch.at.buildtime/ > +MAKE_ENV += GOCACHE="${MODGO_GOCACHE}" > + > MODGO_CMD ?= ${SETENV} ${MAKE_ENV} go > MODGO_BUILD_CMD = ${MODGO_CMD} install ${MODGO_FLAGS} > MODGO_TEST_CMD = ${MODGO_CMD} test ${MODGO_FLAGS} ${MODGO_TEST_FLAGS} > @@ -54,19 +56,38 @@ MODGO_BUILD_CMD += -ldflags="${MODGO_LDFLAGS}" > MODGO_TEST_CMD += -ldflags="${MODGO_LDFLAGS}" > .endif > > -.if defined(GH_ACCOUNT) && defined(GH_PROJECT) > -ALL_TARGET ?= github.com/${GH_ACCOUNT}/${GH_PROJECT} > +.if defined(MODGO_MODNAME) > +EXTRACT_SUFX ?= .zip > +PKGNAME ?= ${DISTNAME:S/-v/-/} > +ALL_TARGET ?= ${MODGO_MODNAME} > +DISTFILES = > ${DISTNAME}${EXTRACT_SUFX}{${MODGO_VERSION}${EXTRACT_SUFX}} > +MASTER_SITES ?= ${MASTER_SITE_ATHENS}${MODGO_MODNAME}/@v/ > +. for _modpath _modver in ${MODGO_MODULES} > +SUPDISTFILES += > ${MODGO_DIST_SUBDIR}/${_modpath}/@v/${_modver}.zip{${_modpath}/@v/${_modver}.zip}:${MODGO_MASTER_SITESN} > +. endfor > +. for _modpath _modver in ${MODGO_MODFILES} > +SUPDISTFILES += > ${MODGO_DIST_SUBDIR}/${_modpath}/@v/${_modver}.mod{${_modpath}/@v/${_modver}.mod}:${MODGO_MASTER_SITESN} > +. endfor > +MAKE_ENV += GOPROXY=file://${DISTDIR}/${MODGO_DIST_SUBDIR} > +MAKE_ENV += GO111MODULE=on GOPATH="${MODGO_GOPATH}" > +.else > +# ports are not allowed to fetch from the network at build time; point > +# GOPROXY at an unreachable host so that failures are also visible to > +# developers who don't have PORTS_PRIVSEP and a "deny .. _pbuild" PF rule. > +MAKE_ENV += GOPROXY=invalid://ports.should.not.fetch.at.buildtime/ > +MAKE_ENV += GO111MODULE=off GOPATH="${MODGO_GOPATH}" > +. if defined(GH_ACCOUNT) && defined(GH_PROJECT) > +ALL_TARGET ?= github.com/${GH_ACCOUNT}/${GH_PROJECT} > +. endif > .endif > -TEST_TARGET ?= ${ALL_TARGET} > > -SEPARATE_BUILD ?= Yes > -WRKSRC ?= ${MODGO_WORKSPACE}/src/${ALL_TARGET} > +MODGO_TEST_TARGET ?= cd ${WRKSRC} && ${MODGO_CMD} test ${ALL_TARGET} > > -MODGO_SETUP_WORKSPACE = mkdir -p ${WRKSRC:H}; mv ${MODGO_SUBDIR} > ${WRKSRC}; > +SEPARATE_BUILD ?= Yes > > CATEGORIES += lang/go > > -MODGO_BUILD_TARGET = ${MODGO_BUILD_CMD} ${ALL_TARGET} > +#MODGO_BUILD_TARGET = ${MODGO_BUILD_CMD} > MODGO_FLAGS += -v -p ${MAKE_JOBS} > > .if empty(DEBUG) > @@ -76,6 +97,14 @@ MODGO_LDFLAGS += -s -w > MODGO_FLAGS += -x > .endif > > +.if empty(MODGO_MODNAME) > +WRKSRC ?= ${MODGO_WORKSPACE}/src/${ALL_TARGET} > +MODGO_SETUP_WORKSPACE = mkdir -p ${WRKSRC:H}; mv ${MODGO_SUBDIR} > ${WRKSRC}; > +.else > +WRKSRC ?= ${WRKDIR}/${MODGO_MODNAME}@${MODGO_VERSION} > +MODGO_SETUP_WORKSPACE = ln -sf ${WRKSRC} ${WRKDIR}/${MODGO_MODNAME} > +.endif > + > INSTALL_STRIP = > .if ${MODGO_TYPE:L:Mbin} > MODGO_INSTALL_TARGET = ${INSTALL_PROGRAM_DIR} > ${PREFIX}/${MODGO_BINDIR} && \ > @@ -99,14 +128,18 @@ MODGO_INSTALL_TARGET += ${INSTALL_DATA_DIR} > ${MODGO_PACKAGE_PATH} && \ > RUN_DEPENDS += ${MODGO_RUN_DEPENDS} > .endif > > -MODGO_TEST_TARGET = ${MODGO_TEST_CMD} ${TEST_TARGET} > - > .if empty(CONFIGURE_STYLE) > MODGO_pre-configure += ${MODGO_SETUP_WORKSPACE} > > -. if !target(do-build) > +. if !target(do-install) > do-build: > - ${MODGO_BUILD_TARGET} > + if [ -d ${WRKSRC}/cmd ]; then \ > + cd ${WRKSRC} && \ > + ${MODGO_BUILD_CMD} ${ALL_TARGET}/cmd/... ; \ > + else \ > + cd ${WRKSRC} && \ > + ${MODGO_BUILD_CMD} ${ALL_TARGET}; \ > + fi; > . endif > > . if !target(do-install) > -- andrew - http://afresh1.com Hey! It compiles! Ship it!