Script 'mail_helper' called by obssrc Hello community, here is the log from the commit of package subfinder for openSUSE:Factory checked in at 2025-03-17 22:18:42 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ Comparing /work/SRC/openSUSE:Factory/subfinder (Old) and /work/SRC/openSUSE:Factory/.subfinder.new.19136 (New) ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Package is "subfinder" Mon Mar 17 22:18:42 2025 rev:4 rq:1253766 version:2.7.0 Changes: -------- --- /work/SRC/openSUSE:Factory/subfinder/subfinder.changes 2025-02-04 18:14:21.772067533 +0100 +++ /work/SRC/openSUSE:Factory/.subfinder.new.19136/subfinder.changes 2025-03-17 22:22:34.946985959 +0100 @@ -1,0 +2,13 @@ +Mon Mar 17 10:16:47 UTC 2025 - Dominik Heidler <dheid...@suse.de> + +- Version 2.7.0 + * Added ThreatCrowd source by @hyni03 in #1508 + * Added DigitalYama source by @nukunga in #1517 + * Fixed issue where input with protocols (e.g., https://example.com ) was skipped. Now, the protocol part is removed, allowing example.com to be used by @dogancanbakir in #1516 + * Fixed VirusTotal limit error where the limit couldn't be larger than 40 by @dogancanbakir in #1548 + * Fixed query syntax for Netlas source by @thurrsense in #1529 + * Removed PassiveTotal as it no longer exists by @dogancanbakir in #1515 + * Changed DNSRepo API endpoint to the latest version by @dogancanbakir in #1540 + * Skipped non-200 HTTP responses for Anubis by @dogancanbakir in #1541 + +------------------------------------------------------------------- Old: ---- subfinder-2.6.8.tar.gz New: ---- subfinder-2.7.0.tar.gz ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ Other differences: ------------------ ++++++ subfinder.spec ++++++ --- /var/tmp/diff_new_pack.SIl4MF/_old 2025-03-17 22:22:36.755061623 +0100 +++ /var/tmp/diff_new_pack.SIl4MF/_new 2025-03-17 22:22:36.759061790 +0100 @@ -17,7 +17,7 @@ Name: subfinder -Version: 2.6.8 +Version: 2.7.0 Release: 0 Summary: Fast passive subdomain enumeration tool License: MIT ++++++ subfinder-2.6.8.tar.gz -> subfinder-2.7.0.tar.gz ++++++ diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/subfinder-2.6.8/.github/workflows/build-test.yml new/subfinder-2.7.0/.github/workflows/build-test.yml --- old/subfinder-2.6.8/.github/workflows/build-test.yml 2025-01-14 10:37:59.000000000 +0100 +++ new/subfinder-2.7.0/.github/workflows/build-test.yml 2025-02-27 08:12:57.000000000 +0100 @@ -51,7 +51,6 @@ HUNTER_API_KEY: ${{secrets.HUNTER_API_KEY}} INTELX_API_KEY: ${{secrets.INTELX_API_KEY}} LEAKIX_API_KEY: ${{secrets.LEAKIX_API_KEY}} - PASSIVETOTAL_API_KEY: ${{secrets.PASSIVETOTAL_API_KEY}} QUAKE_API_KEY: ${{secrets.QUAKE_API_KEY}} ROBTEX_API_KEY: ${{secrets.ROBTEX_API_KEY}} SECURITYTRAILS_API_KEY: ${{secrets.SECURITYTRAILS_API_KEY}} diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/subfinder-2.6.8/v2/go.mod new/subfinder-2.7.0/v2/go.mod --- old/subfinder-2.6.8/v2/go.mod 2025-01-14 10:37:59.000000000 +0100 +++ new/subfinder-2.7.0/v2/go.mod 2025-02-27 08:12:57.000000000 +0100 @@ -8,12 +8,12 @@ github.com/json-iterator/go v1.1.12 github.com/lib/pq v1.10.9 github.com/projectdiscovery/chaos-client v0.5.2 - github.com/projectdiscovery/dnsx v1.2.1 + github.com/projectdiscovery/dnsx v1.2.2 github.com/projectdiscovery/fdmax v0.0.4 - github.com/projectdiscovery/gologger v1.1.40 - github.com/projectdiscovery/ratelimit v0.0.68 - github.com/projectdiscovery/retryablehttp-go v1.0.95 - github.com/projectdiscovery/utils v0.4.6 + github.com/projectdiscovery/gologger v1.1.44 + github.com/projectdiscovery/ratelimit v0.0.70 + github.com/projectdiscovery/retryablehttp-go v1.0.99 + github.com/projectdiscovery/utils v0.4.11 github.com/rs/xid v1.5.0 github.com/stretchr/testify v1.9.0 github.com/tomnomnom/linkheader v0.0.0-20180905144013-02ca5825eb80 @@ -71,10 +71,10 @@ github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect github.com/projectdiscovery/blackrock v0.0.1 // indirect github.com/projectdiscovery/cdncheck v1.1.0 // indirect - github.com/projectdiscovery/fastdialer v0.2.14 // indirect - github.com/projectdiscovery/hmap v0.0.74 // indirect + github.com/projectdiscovery/fastdialer v0.3.0 // indirect + github.com/projectdiscovery/hmap v0.0.80 // indirect github.com/projectdiscovery/machineid v0.0.0-20240226150047-2e2c51e35983 // indirect - github.com/projectdiscovery/networkpolicy v0.0.9 // indirect + github.com/projectdiscovery/networkpolicy v0.1.1 // indirect github.com/refraction-networking/utls v1.6.7 // indirect github.com/rivo/uniseg v0.4.7 // indirect github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d // indirect @@ -124,8 +124,8 @@ github.com/modern-go/reflect2 v1.0.2 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect - github.com/projectdiscovery/goflags v0.1.64 - github.com/projectdiscovery/retryabledns v1.0.93 // indirect + github.com/projectdiscovery/goflags v0.1.72 + github.com/projectdiscovery/retryabledns v1.0.94 // indirect golang.org/x/net v0.33.0 // indirect golang.org/x/sys v0.28.0 // indirect ) diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/subfinder-2.6.8/v2/go.sum new/subfinder-2.7.0/v2/go.sum --- old/subfinder-2.6.8/v2/go.sum 2025-01-14 10:37:59.000000000 +0100 +++ new/subfinder-2.7.0/v2/go.sum 2025-02-27 08:12:57.000000000 +0100 @@ -194,38 +194,39 @@ github.com/projectdiscovery/cdncheck v1.1.0/go.mod h1:sZ8U4MjHSsyaTVjBbYWHT1cwUVvUYwDX1W+WvWRicIc= github.com/projectdiscovery/chaos-client v0.5.2 h1:dN+7GXEypsJAbCD//dBcUxzAEAEH1fjc/7Rf4F/RiNU= github.com/projectdiscovery/chaos-client v0.5.2/go.mod h1:KnoJ/NJPhll42uaqlDga6oafFfNw5l2XI2ajRijtDuU= -github.com/projectdiscovery/dnsx v1.2.1 h1:TxslYvp1Z/YZ4CP/J0gx5RYpvXREnVmyoacmTcGu5yg= -github.com/projectdiscovery/dnsx v1.2.1/go.mod h1:6dAsMCEDu7FArZy2qjyTeUQrqpZ4ITLU11fcmUvFqt0= -github.com/projectdiscovery/fastdialer v0.2.14 h1:/cndy+5celjoYzbk4LksHYOCTpFGIJY8RF/EK31Opjs= -github.com/projectdiscovery/fastdialer v0.2.14/go.mod h1:z5yKQ/YWaVrBMfdL6f5J7VytUx9wxc5vs/Lf51QelCw= +github.com/projectdiscovery/dnsx v1.2.2 h1:ZjUov0GOyrS8ERlKAAhk+AOkqzaYHBzCP0qZfO+6Ihg= +github.com/projectdiscovery/dnsx v1.2.2/go.mod h1:3iYm86OEqo0WxeGDkVl5WZNmG0qYE5TYNx8fBg6wX1I= +github.com/projectdiscovery/fastdialer v0.3.0 h1:/wMptjdsrAU/wiaA/U3lSgYGaYCGJH6xm0mLei6oMxk= +github.com/projectdiscovery/fastdialer v0.3.0/go.mod h1:Q0YLArvpx9GAfY/NcTPMCA9qZuVOGnuVoNYWzKBwxdQ= github.com/projectdiscovery/fdmax v0.0.4 h1:K9tIl5MUZrEMzjvwn/G4drsHms2aufTn1xUdeVcmhmc= github.com/projectdiscovery/fdmax v0.0.4/go.mod h1:oZLqbhMuJ5FmcoaalOm31B1P4Vka/CqP50nWjgtSz+I= -github.com/projectdiscovery/goflags v0.1.64 h1:FDfwdt9N97Hi8OuhbkDlKtVttpc/CRMIWQVa08VsHsI= -github.com/projectdiscovery/goflags v0.1.64/go.mod h1:3FyHIVQtnycNOc1LE3O1jj/XR5XuMdF9QfHd0ujhnX4= -github.com/projectdiscovery/gologger v1.1.40 h1:FSIhKnYKzuIEIz3RTg6JX9JtDKgkEzIEf2v5RYckoQ4= -github.com/projectdiscovery/gologger v1.1.40/go.mod h1:8AUxYXmClqOWJgZ5wknNn5rRK3UlrXQ/r9JjX+gp5Gg= -github.com/projectdiscovery/hmap v0.0.74 h1:j0TpS9fJxisfdGcIRjaZ3qgyjP3pBkRJf12ZWv64rOI= -github.com/projectdiscovery/hmap v0.0.74/go.mod h1:qEPAdq/gWQU/IEI+QMzSyL+HYdqayR64V9vGTI/W38c= +github.com/projectdiscovery/goflags v0.1.72 h1:tSR+BnfDLbfTGYYVg4k1oQcFOoYXPY1pllV0MHtx3ek= +github.com/projectdiscovery/goflags v0.1.72/go.mod h1:C2cZ+PJRx7bbArEp/qFUixjsYFDd3etFNNHMUdJqfr8= +github.com/projectdiscovery/gologger v1.1.44 h1:tprWkKzKt37pz4HG2tvhzrOCQNIn8A3CEki6BRzXE5o= +github.com/projectdiscovery/gologger v1.1.44/go.mod h1:ZQS0eJq7BwKM0xxFqwZFUkAH1bkIqe90EOFBP4LENH4= +github.com/projectdiscovery/hmap v0.0.80 h1:2PSo3qQNKanK6i6DF4NzsVEJANe6tMIBmBtxvF4AKK8= +github.com/projectdiscovery/hmap v0.0.80/go.mod h1:YmZ9qwtl7MDJYrpxJ+MEw4N4V58w18WGPsQHgpdIV0s= github.com/projectdiscovery/machineid v0.0.0-20240226150047-2e2c51e35983 h1:ZScLodGSezQVwsQDtBSMFp72WDq0nNN+KE/5DHKY5QE= github.com/projectdiscovery/machineid v0.0.0-20240226150047-2e2c51e35983/go.mod h1:3G3BRKui7nMuDFAZKR/M2hiOLtaOmyukT20g88qRQjI= -github.com/projectdiscovery/networkpolicy v0.0.9 h1:IrlDoYZagNNO8y+7iZeHT8k5izE+nek7TdtvEBwCxqk= -github.com/projectdiscovery/networkpolicy v0.0.9/go.mod h1:XFJ2Lnv8BE/ziQCFjBHMsH1w6VmkPiQtk+NlBpdMU7M= -github.com/projectdiscovery/ratelimit v0.0.68 h1:gMLD1aB4R8w7BIpKvtQf6TNb6+5zsJO9WSRWZ9pxwe4= -github.com/projectdiscovery/ratelimit v0.0.68/go.mod h1:ieU9nNu9Ie8nVMKdj3bsX3JA3kfNI8qn4pkNXsyRxsw= -github.com/projectdiscovery/retryabledns v1.0.93 h1:iKcEEEH77WwUf5EGimhHxCDdqBF2kOl7WhQi3VQXB8Q= -github.com/projectdiscovery/retryabledns v1.0.93/go.mod h1:f5HmPdVr3CUm4tHHiB0UyiZVQTYYAKTqfoj8M2gCvqo= -github.com/projectdiscovery/retryablehttp-go v1.0.95 h1:5CHhWLMovX1MD9W3HzlsMBY3xA+dyeqta2gSWo3j92E= -github.com/projectdiscovery/retryablehttp-go v1.0.95/go.mod h1:/7CHaD7vqnqBD++AI0JsJdcYyq1Wbf4vMhddjy7sZjI= -github.com/projectdiscovery/utils v0.4.6 h1:lwbS5d/f70wyDwuwF6lAVkn390hEI/0LOtqyqJEI+qE= -github.com/projectdiscovery/utils v0.4.6/go.mod h1:eevtW7+x7ydrBdmOenmHdqqJKRv3VqY2QUR7vs4qRfU= +github.com/projectdiscovery/networkpolicy v0.1.1 h1:iv9gECukD5KAZp98KVh+T3TEPTkY6dr3sKsdbh9XyZU= +github.com/projectdiscovery/networkpolicy v0.1.1/go.mod h1:/Hg2ieLewSe/BagFF+UYXAQo3NwmVMq16MSAl492XkU= +github.com/projectdiscovery/ratelimit v0.0.70 h1:SxFQcIKO3hppmEn9MOaDiqX2NXceji0vd8ER+eCHQjc= +github.com/projectdiscovery/ratelimit v0.0.70/go.mod h1:jg253i7eeKBIV5QpTpQv6+lZXr53XmKGBLS3dwlmRWM= +github.com/projectdiscovery/retryabledns v1.0.94 h1:MvxtRcmvxhxikxT7p/E40hcYRWRiL5fg/JQ8bpBaz+0= +github.com/projectdiscovery/retryabledns v1.0.94/go.mod h1:croGTyMM4yNlrSWA/X7xNe3c0c7mDmCdbm8goLd8Bak= +github.com/projectdiscovery/retryablehttp-go v1.0.99 h1:S+lQqo1ZnO5aoWsBV8HapGslJSaYVUII954SnH1RSjw= +github.com/projectdiscovery/retryablehttp-go v1.0.99/go.mod h1:8Mv9L9vjmam16garE6/dqLFkT0ZcfLNSo9O1zFBiPlE= +github.com/projectdiscovery/utils v0.4.11 h1:MWqCFxYINQPa4KWMRNah7W0N1COGRhqOpGVhiR/VaO0= +github.com/projectdiscovery/utils v0.4.11/go.mod h1:47tvqErksJELcxDBH8An2i9qvUe5E1qR7B72xxqiyqU= github.com/refraction-networking/utls v1.6.7 h1:zVJ7sP1dJx/WtVuITug3qYUq034cDq9B2MR1K67ULZM= github.com/refraction-networking/utls v1.6.7/go.mod h1:BC3O4vQzye5hqpmDTWUqi4P5DDhzJfkV1tdqtawQIH0= github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ= github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= -github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= +github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= +github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= github.com/rs/xid v1.5.0 h1:mKX4bl4iPYJtEIxp6CYiUuLQ/8DYMoz0PUdtGgMFRVc= github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d h1:hrujxIzL1woJ7AwssoOcM/tq5JjjG2yYOc8odClEiXA= diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/subfinder-2.6.8/v2/pkg/passive/sources.go new/subfinder-2.7.0/v2/pkg/passive/sources.go --- old/subfinder-2.6.8/v2/pkg/passive/sources.go 2025-01-14 10:37:59.000000000 +0100 +++ new/subfinder-2.7.0/v2/pkg/passive/sources.go 2025-02-27 08:12:57.000000000 +0100 @@ -34,7 +34,6 @@ "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/intelx" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/leakix" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/netlas" - "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/passivetotal" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/quake" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/rapiddns" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/redhuntlabs" @@ -43,10 +42,12 @@ "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/shodan" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/sitedossier" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/threatbook" + "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/threatcrowd" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/virustotal" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/waybackarchive" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/whoisxmlapi" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/zoomeyeapi" + "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/digitalyama" mapsutil "github.com/projectdiscovery/utils/maps" ) @@ -75,7 +76,6 @@ &intelx.Source{}, &netlas.Source{}, &leakix.Source{}, - &passivetotal.Source{}, &quake.Source{}, &rapiddns.Source{}, &redhuntlabs.Source{}, @@ -85,6 +85,7 @@ &shodan.Source{}, &sitedossier.Source{}, &threatbook.Source{}, + &threatcrowd.Source{}, &virustotal.Source{}, &waybackarchive.Source{}, &whoisxmlapi.Source{}, @@ -94,12 +95,11 @@ // &reconcloud.Source{}, // failing due to cloudflare bot protection &builtwith.Source{}, &hudsonrock.Source{}, + &digitalyama.Source{}, } var sourceWarnings = mapsutil.NewSyncLockMap[string, string]( - mapsutil.WithMap(mapsutil.Map[string, string]{ - "passivetotal": "New API credentials for PassiveTotal can't be generated, but existing user account credentials are still functional. Please ensure your integrations are using valid credentials.", - })) + mapsutil.WithMap(mapsutil.Map[string, string]{})) var NameSourceMap = make(map[string]subscraping.Source, len(AllSources)) diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/subfinder-2.6.8/v2/pkg/passive/sources_test.go new/subfinder-2.7.0/v2/pkg/passive/sources_test.go --- old/subfinder-2.6.8/v2/pkg/passive/sources_test.go 2025-01-14 10:37:59.000000000 +0100 +++ new/subfinder-2.7.0/v2/pkg/passive/sources_test.go 2025-02-27 08:12:57.000000000 +0100 @@ -33,7 +33,6 @@ "hackertarget", "intelx", "netlas", - "passivetotal", "quake", "rapiddns", "redhuntlabs", @@ -43,6 +42,7 @@ "shodan", "sitedossier", "threatbook", + "threatcrowd", "virustotal", "waybackarchive", "whoisxmlapi", @@ -54,6 +54,7 @@ // "reconcloud", "builtwith", "hudsonrock", + "digitalyama", } expectedDefaultSources = []string{ @@ -74,7 +75,6 @@ "fullhunt", "hackertarget", "intelx", - "passivetotal", "quake", "redhuntlabs", "robtex", @@ -89,6 +89,7 @@ // "threatminer", // "reconcloud", "builtwith", + "digitalyama", } expectedDefaultRecursiveSources = []string{ @@ -100,7 +101,6 @@ "dnsdb", "digitorus", "hackertarget", - "passivetotal", "securitytrails", "virustotal", "leakix", diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/subfinder-2.6.8/v2/pkg/runner/banners.go new/subfinder-2.7.0/v2/pkg/runner/banners.go --- old/subfinder-2.6.8/v2/pkg/runner/banners.go 2025-01-14 10:37:59.000000000 +0100 +++ new/subfinder-2.7.0/v2/pkg/runner/banners.go 2025-02-27 08:12:57.000000000 +0100 @@ -17,7 +17,7 @@ const ToolName = `subfinder` // Version is the current version of subfinder -const version = `v2.6.8` +const version = `v2.7.0` // showBanner is used to show the banner to the user func showBanner() { diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/subfinder-2.6.8/v2/pkg/runner/enumerate.go new/subfinder-2.7.0/v2/pkg/runner/enumerate.go --- old/subfinder-2.6.8/v2/pkg/runner/enumerate.go 2025-01-14 10:37:59.000000000 +0100 +++ new/subfinder-2.7.0/v2/pkg/runner/enumerate.go 2025-02-27 08:12:57.000000000 +0100 @@ -19,6 +19,7 @@ const maxNumCount = 2 var replacer = strings.NewReplacer( + "/", "", "â¢.", "", "â¢", "", "*.", "", diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/subfinder-2.6.8/v2/pkg/runner/runner.go new/subfinder-2.7.0/v2/pkg/runner/runner.go --- old/subfinder-2.6.8/v2/pkg/runner/runner.go 2025-01-14 10:37:59.000000000 +0100 +++ new/subfinder-2.7.0/v2/pkg/runner/runner.go 2025-02-27 08:12:57.000000000 +0100 @@ -119,6 +119,7 @@ ip, _ := regexp.Compile(`^([0-9\.]+$)`) for scanner.Scan() { domain := preprocessDomain(scanner.Text()) + domain = replacer.Replace(domain) if domain == "" || (r.options.ExcludeIps && ip.MatchString(domain)) { continue diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/subfinder-2.6.8/v2/pkg/subscraping/sources/anubis/anubis.go new/subfinder-2.7.0/v2/pkg/subscraping/sources/anubis/anubis.go --- old/subfinder-2.6.8/v2/pkg/subscraping/sources/anubis/anubis.go 2025-01-14 10:37:59.000000000 +0100 +++ new/subfinder-2.7.0/v2/pkg/subscraping/sources/anubis/anubis.go 2025-02-27 08:12:57.000000000 +0100 @@ -4,6 +4,7 @@ import ( "context" "fmt" + "net/http" "time" jsoniter "github.com/json-iterator/go" @@ -38,6 +39,11 @@ return } + if resp.StatusCode != http.StatusOK { + resp.Body.Close() + return + } + var subdomains []string err = jsoniter.NewDecoder(resp.Body).Decode(&subdomains) if err != nil { diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/subfinder-2.6.8/v2/pkg/subscraping/sources/digitalyama/digitalyama.go new/subfinder-2.7.0/v2/pkg/subscraping/sources/digitalyama/digitalyama.go --- old/subfinder-2.6.8/v2/pkg/subscraping/sources/digitalyama/digitalyama.go 1970-01-01 01:00:00.000000000 +0100 +++ new/subfinder-2.7.0/v2/pkg/subscraping/sources/digitalyama/digitalyama.go 2025-02-27 08:12:57.000000000 +0100 @@ -0,0 +1,128 @@ +package digitalyama + +import ( + "context" + "fmt" + "time" + + jsoniter "github.com/json-iterator/go" + + "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" +) + +// Source is the passive scraping agent +type Source struct { + apiKeys []string + timeTaken time.Duration + errors int + results int + skipped bool +} + +type digitalYamaResponse struct { + Query string `json:"query"` + Count int `json:"count"` + Subdomains []string `json:"subdomains"` + UsageSummary struct { + QueryCost float64 `json:"query_cost"` + CreditsRemaining float64 `json:"credits_remaining"` + } `json:"usage_summary"` +} + +// Run function returns all subdomains found with the service +func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { + results := make(chan subscraping.Result) + s.errors = 0 + s.results = 0 + + go func() { + defer func(startTime time.Time) { + s.timeTaken = time.Since(startTime) + close(results) + }(time.Now()) + + randomApiKey := subscraping.PickRandom(s.apiKeys, s.Name()) + if randomApiKey == "" { + s.skipped = true + return + } + + searchURL := fmt.Sprintf("https://api.digitalyama.com/subdomain_finder?domain=%s";, domain) + resp, err := session.Get(ctx, searchURL, "", map[string]string{"x-api-key": randomApiKey}) + if err != nil { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + s.errors++ + return + } + defer resp.Body.Close() + + if resp.StatusCode != 200 { + var errResponse struct { + Detail []struct { + Loc []string `json:"loc"` + Msg string `json:"msg"` + Type string `json:"type"` + } `json:"detail"` + } + err = jsoniter.NewDecoder(resp.Body).Decode(&errResponse) + if err != nil { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: fmt.Errorf("unexpected status code %d", resp.StatusCode)} + s.errors++ + return + } + if len(errResponse.Detail) > 0 { + errMsg := errResponse.Detail[0].Msg + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: fmt.Errorf("%s (code %d)", errMsg, resp.StatusCode)} + } else { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: fmt.Errorf("unexpected status code %d", resp.StatusCode)} + } + s.errors++ + return + } + + var response digitalYamaResponse + err = jsoniter.NewDecoder(resp.Body).Decode(&response) + if err != nil { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + s.errors++ + return + } + + for _, subdomain := range response.Subdomains { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain} + s.results++ + } + }() + + return results +} + +// Name returns the name of the source +func (s *Source) Name() string { + return "digitalyama" +} + +func (s *Source) IsDefault() bool { + return true +} + +func (s *Source) HasRecursiveSupport() bool { + return false +} + +func (s *Source) NeedsKey() bool { + return true +} + +func (s *Source) AddApiKeys(keys []string) { + s.apiKeys = keys +} + +func (s *Source) Statistics() subscraping.Statistics { + return subscraping.Statistics{ + Errors: s.errors, + Results: s.results, + TimeTaken: s.timeTaken, + Skipped: s.skipped, + } +} diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/subfinder-2.6.8/v2/pkg/subscraping/sources/dnsrepo/dnsrepo.go new/subfinder-2.7.0/v2/pkg/subscraping/sources/dnsrepo/dnsrepo.go --- old/subfinder-2.6.8/v2/pkg/subscraping/sources/dnsrepo/dnsrepo.go 2025-01-14 10:37:59.000000000 +0100 +++ new/subfinder-2.7.0/v2/pkg/subscraping/sources/dnsrepo/dnsrepo.go 2025-02-27 08:12:57.000000000 +0100 @@ -21,7 +21,7 @@ } type DnsRepoResponse []struct { - Domain string + Domain string `json:"domain"` } func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { @@ -40,7 +40,17 @@ s.skipped = true return } - resp, err := session.SimpleGet(ctx, fmt.Sprintf("https://dnsrepo.noc.org/api/?apikey=%s&search=%s";, randomApiKey, domain)) + + randomApiInfo := strings.Split(randomApiKey, ":") + if len(randomApiInfo) != 2 { + s.skipped = true + return + } + + token := randomApiInfo[0] + apiKey := randomApiInfo[1] + + resp, err := session.Get(ctx, fmt.Sprintf("https://dnsarchive.net/api/?apikey=%s&search=%s";, apiKey, domain), "", map[string]string{"X-API-Access": token}) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} s.errors++ diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/subfinder-2.6.8/v2/pkg/subscraping/sources/netlas/netlas.go new/subfinder-2.7.0/v2/pkg/subscraping/sources/netlas/netlas.go --- old/subfinder-2.6.8/v2/pkg/subscraping/sources/netlas/netlas.go 2025-01-14 10:37:59.000000000 +0100 +++ new/subfinder-2.7.0/v2/pkg/subscraping/sources/netlas/netlas.go 2025-02-27 08:12:57.000000000 +0100 @@ -98,10 +98,10 @@ // Make a single POST request to get all domains via download method apiUrl := "https://app.netlas.io/api/domains/download/"; - query := fmt.Sprintf("domain:(domain:*.%s AND NOT domain:%s)", domain, domain) + query := fmt.Sprintf("domain:*.%s AND NOT domain:%s", domain, domain) requestBody := map[string]interface{}{ - "q": query, - "fields": []string{"*"}, + "q": query, + "fields": []string{"*"}, "source_type": "include", "size": domainsCount.Count, } @@ -116,8 +116,8 @@ randomApiKey = subscraping.PickRandom(s.apiKeys, s.Name()) resp, err = session.HTTPRequest(ctx, http.MethodPost, apiUrl, "", map[string]string{ - "accept": "application/json", - "X-API-Key": randomApiKey, + "accept": "application/json", + "X-API-Key": randomApiKey, "Content-Type": "application/json"}, strings.NewReader(string(jsonRequestBody)), subscraping.BasicAuth{}) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/subfinder-2.6.8/v2/pkg/subscraping/sources/passivetotal/passivetotal.go new/subfinder-2.7.0/v2/pkg/subscraping/sources/passivetotal/passivetotal.go --- old/subfinder-2.6.8/v2/pkg/subscraping/sources/passivetotal/passivetotal.go 2025-01-14 10:37:59.000000000 +0100 +++ new/subfinder-2.7.0/v2/pkg/subscraping/sources/passivetotal/passivetotal.go 1970-01-01 01:00:00.000000000 +0100 @@ -1,126 +0,0 @@ -// Package passivetotal logic -package passivetotal - -import ( - "bytes" - "context" - "regexp" - "time" - - jsoniter "github.com/json-iterator/go" - - "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" -) - -var passiveTotalFilterRegex = regexp.MustCompile(`^(?:\d{1,3}\.){3}\d{1,3}\\032`) - -type response struct { - Subdomains []string `json:"subdomains"` -} - -// Source is the passive scraping agent -type Source struct { - apiKeys []apiKey - timeTaken time.Duration - errors int - results int - skipped bool -} - -type apiKey struct { - username string - password string -} - -// Run function returns all subdomains found with the service -func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { - results := make(chan subscraping.Result) - s.errors = 0 - s.results = 0 - - go func() { - defer func(startTime time.Time) { - s.timeTaken = time.Since(startTime) - close(results) - }(time.Now()) - - randomApiKey := subscraping.PickRandom(s.apiKeys, s.Name()) - if randomApiKey.username == "" || randomApiKey.password == "" { - s.skipped = true - return - } - - // Create JSON Get body - var request = []byte(`{"query":"` + domain + `"}`) - - resp, err := session.HTTPRequest( - ctx, - "GET", - "https://api.passivetotal.org/v2/enrichment/subdomains";, - "", - map[string]string{"Content-Type": "application/json"}, - bytes.NewBuffer(request), - subscraping.BasicAuth{Username: randomApiKey.username, Password: randomApiKey.password}, - ) - if err != nil { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} - s.errors++ - session.DiscardHTTPResponse(resp) - return - } - - var data response - err = jsoniter.NewDecoder(resp.Body).Decode(&data) - if err != nil { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} - s.errors++ - resp.Body.Close() - return - } - resp.Body.Close() - - for _, subdomain := range data.Subdomains { - // skip entries like xxx.xxx.xxx.xxx\032domain.tld - if passiveTotalFilterRegex.MatchString(subdomain) { - continue - } - finalSubdomain := subdomain + "." + domain - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: finalSubdomain} - s.results++ - } - }() - - return results -} - -// Name returns the name of the source -func (s *Source) Name() string { - return "passivetotal" -} - -func (s *Source) IsDefault() bool { - return true -} - -func (s *Source) HasRecursiveSupport() bool { - return true -} - -func (s *Source) NeedsKey() bool { - return true -} - -func (s *Source) AddApiKeys(keys []string) { - s.apiKeys = subscraping.CreateApiKeys(keys, func(k, v string) apiKey { - return apiKey{k, v} - }) -} - -func (s *Source) Statistics() subscraping.Statistics { - return subscraping.Statistics{ - Errors: s.errors, - Results: s.results, - TimeTaken: s.timeTaken, - Skipped: s.skipped, - } -} diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/subfinder-2.6.8/v2/pkg/subscraping/sources/threatcrowd/threatcrowd.go new/subfinder-2.7.0/v2/pkg/subscraping/sources/threatcrowd/threatcrowd.go --- old/subfinder-2.6.8/v2/pkg/subscraping/sources/threatcrowd/threatcrowd.go 1970-01-01 01:00:00.000000000 +0100 +++ new/subfinder-2.7.0/v2/pkg/subscraping/sources/threatcrowd/threatcrowd.go 2025-02-27 08:12:57.000000000 +0100 @@ -0,0 +1,117 @@ +package threatcrowd + +import ( + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "time" + + "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" +) + +// threatCrowdResponse represents the JSON response from the ThreatCrowd API. +type threatCrowdResponse struct { + ResponseCode string `json:"response_code"` + Subdomains []string `json:"subdomains"` + Undercount string `json:"undercount"` +} + +// Source implements the subscraping.Source interface for ThreatCrowd. +type Source struct { + timeTaken time.Duration + errors int + results int +} + +// Run queries the ThreatCrowd API for the given domain and returns found subdomains. +func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { + results := make(chan subscraping.Result) + s.errors = 0 + s.results = 0 + + go func(startTime time.Time) { + defer func() { + s.timeTaken = time.Since(startTime) + close(results) + }() + + url := fmt.Sprintf("http://ci-www.threatcrowd.org/searchApi/v2/domain/report/?domain=%s";, domain) + req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) + if err != nil { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + s.errors++ + return + } + + resp, err := session.Client.Do(req) + if err != nil { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + s.errors++ + return + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: fmt.Errorf("unexpected status code: %d", resp.StatusCode)} + s.errors++ + return + } + + body, err := io.ReadAll(resp.Body) + if err != nil { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + s.errors++ + return + } + + var tcResponse threatCrowdResponse + if err := json.Unmarshal(body, &tcResponse); err != nil { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + s.errors++ + return + } + + for _, subdomain := range tcResponse.Subdomains { + if subdomain != "" { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain} + s.results++ + } + } + }(time.Now()) + + return results +} + +// Name returns the name of the source. +func (s *Source) Name() string { + return "threatcrowd" +} + +// IsDefault indicates whether this source is enabled by default. +func (s *Source) IsDefault() bool { + return false +} + +// HasRecursiveSupport indicates if the source supports recursive searches. +func (s *Source) HasRecursiveSupport() bool { + return false +} + +// NeedsKey indicates if the source requires an API key. +func (s *Source) NeedsKey() bool { + return false +} + +// AddApiKeys is a no-op since ThreatCrowd does not require an API key. +func (s *Source) AddApiKeys(_ []string) {} + +// Statistics returns usage statistics. +func (s *Source) Statistics() subscraping.Statistics { + return subscraping.Statistics{ + Errors: s.errors, + Results: s.results, + TimeTaken: s.timeTaken, + } +} diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/subfinder-2.6.8/v2/pkg/subscraping/sources/virustotal/virustotal.go new/subfinder-2.7.0/v2/pkg/subscraping/sources/virustotal/virustotal.go --- old/subfinder-2.6.8/v2/pkg/subscraping/sources/virustotal/virustotal.go 2025-01-14 10:37:59.000000000 +0100 +++ new/subfinder-2.7.0/v2/pkg/subscraping/sources/virustotal/virustotal.go 2025-02-27 08:12:57.000000000 +0100 @@ -51,7 +51,7 @@ } var cursor string = "" for { - var url string = fmt.Sprintf("https://www.virustotal.com/api/v3/domains/%s/subdomains?limit=1000";, domain) + var url string = fmt.Sprintf("https://www.virustotal.com/api/v3/domains/%s/subdomains?limit=40";, domain) if cursor != "" { url = fmt.Sprintf("%s&cursor=%s", url, cursor) } ++++++ vendor.tar.zstd ++++++ Binary files /var/tmp/diff_new_pack.SIl4MF/_old and /var/tmp/diff_new_pack.SIl4MF/_new differ
