Skip to content

Commit

Permalink
feat: fetch ranges from AS via bgp tools (#8)
Browse files Browse the repository at this point in the history
* new internal util function for sources: `getRangesForAsn`, which allows to get all ranges (CIDRs) for a given ASN using <https://bgp.tools/table.txt>

Following providers are now sourced from this technique
* Alibaba
* Ovh
* Scaleway
* Tencent
* Ucloud
  • Loading branch information
nohehf authored Oct 2, 2024
1 parent 1583f6c commit e28dba3
Show file tree
Hide file tree
Showing 18 changed files with 2,567 additions and 3,459 deletions.
26 changes: 13 additions & 13 deletions internal/source/source_alibaba.go
Original file line number Diff line number Diff line change
Expand Up @@ -7,26 +7,26 @@ import (

type Alibaba struct{}

const alibabaFileURL = "https://raw.githubusercontent.com/devanshbatham/ip2cloud/main/data/aliyun.txt"
var AlibabaASNs = []string{
// Alibaba cloud
"24429",
// Alibaba AS45102
"45102",
// Alibaba (china) AS37963
"37963",
}

func (a Alibaba) GetProvider() provider.Provider {
return provider.Alibaba
}

func (a Alibaba) GetIPRanges() []*IPRange {
log.Info("Using static Alibaba ip ranges")

ranges := make([]*IPRange, 0)
alibabaRanges, err := LoadTextURLToRange(alibabaFileURL)
if err != nil {
log.Fatal("Failed to load text url to range for Alibaba", err)
}
for _, cidr := range alibabaRanges {
network, cat := ParseCIDR(cidr)
ranges = append(ranges, &IPRange{
Network: network,
Cat: cat,
})
for _, asn := range AlibabaASNs {
log.Info("[Alibaba] - Using ranges from ASN list (AS%s)", asn)
_ranges := getRangesForAsn(asn)
ranges = append(ranges, _ranges...)
log.Info("[Alibaba] - Found %d ranges for AS%s", len(ranges), asn)
}
return ranges
}
26 changes: 12 additions & 14 deletions internal/source/source_linode.go
Original file line number Diff line number Diff line change
Expand Up @@ -7,26 +7,24 @@ import (

type Linode struct{}

const LinodeFileURL = "https://raw.githubusercontent.com/devanshbatham/ip2cloud/main/data/linode.txt"

func (a Linode) GetProvider() provider.Provider {
return provider.Linode
}

func (a Linode) GetIPRanges() []*IPRange {
log.Info("Using static Linode ip ranges")
var LinodeASNs = []string{
// Linode AS63949
"63949",
// Linode CorpNet
"48337",
}

func (a Linode) GetIPRanges() []*IPRange {
ranges := make([]*IPRange, 0)
linodeRanges, err := LoadTextURLToRange(LinodeFileURL)
if err != nil {
log.Fatal("Failed to load text url to range for Linode", err)
}
for _, cidr := range linodeRanges {
network, cat := ParseCIDR(cidr)
ranges = append(ranges, &IPRange{
Network: network,
Cat: cat,
})
for _, asn := range LinodeASNs {
log.Info("[Linode] - Using ranges from ASN list (AS%s)", asn)
_ranges := getRangesForAsn(asn)
ranges = append(ranges, _ranges...)
log.Info("[Linode] - Found %d ranges for AS%s", len(ranges), asn)
}
return ranges
}
76 changes: 6 additions & 70 deletions internal/source/source_ovh.go
Original file line number Diff line number Diff line change
Expand Up @@ -7,80 +7,16 @@ import (

type Ovh struct{}

// TODO: Dynamically fetch the sources
// Source: https://ipinfo.io/AS16276#block-ranges
var ovhRanges = [...]string{
"135.125.0.0/17",
"135.125.128.0/17",
"135.148.0.0/17",
"135.148.128.0/17",
"137.74.0.0/16",
"139.99.0.0/17",
"139.99.128.0/17",
"141.227.128.0/24",
"141.227.130.0/24",
"141.227.132.0/24",
"141.227.134.0/24",
"141.227.136.0/24",
"141.227.138.0/24",
"141.227.140.0/24",
"141.94.0.0/16",
"141.95.0.0/17",
"141.95.128.0/17",
"142.4.192.0/19",
"142.44.128.0/17",
"142.44.140.0/24",
"144.217.0.0/16",
"145.239.0.0/16",
"146.59.0.0/16",
"146.59.0.0/17",
"147.135.0.0/17",
"147.135.128.0/17",
"148.113.0.0/18",
"148.113.128.0/17",
"149.202.0.0/16",
"149.56.0.0/16",
"151.80.0.0/16",
"15.204.0.0/17",
"15.204.128.0/17",
"152.228.128.0/17",
"15.235.0.0/17",
"15.235.128.0/17",
"158.69.0.0/16",
"162.19.0.0/17",
"162.19.128.0/17",
"164.132.0.0/16",
"167.114.0.0/17",
"167.114.128.0/18",
"167.114.192.0/19",
"176.31.0.0/16",
"178.32.0.0/15",
"185.15.68.0/22",
"185.45.160.0/22",
"188.165.0.0/16",
"192.240.152.0/21",
"192.31.246.0/24",
"192.95.0.0/18",
"192.99.0.0/16",
"192.99.65.0/24",
"193.70.0.0/17",
}

func (a Ovh) GetProvider() provider.Provider {
return provider.Ovh
}

func (a Ovh) GetIPRanges() []*IPRange {
log.Info("Using static ovh ip ranges")

ranges := make([]*IPRange, 0)
for _, cidr := range ovhRanges {
network, cat := ParseCIDR(cidr)
ranges = append(ranges, &IPRange{
Network: network,
Cat: cat,
})
}
// Note: there is also AS22598 for OVHTelecom. But this probably doesn't expose hosting services
var OvhCloudASN = "16276"

func (a Ovh) GetIPRanges() []*IPRange {
log.Info("[Ovh] - Using ranges from ASN list (AS%s)", OvhCloudASN)
ranges := getRangesForAsn(OvhCloudASN)
log.Info("[Ovh] - Found %d ranges for AS%s", len(ranges), OvhCloudASN)
return ranges
}
35 changes: 5 additions & 30 deletions internal/source/source_scaleway.go
Original file line number Diff line number Diff line change
Expand Up @@ -7,40 +7,15 @@ import (

type Scaleway struct{}

// Source: https://ipinfo.io/AS12876#block-ranges
var scalewayRanges = [...]string{
"151.115.0.0/18",
"151.115.64.0/18",
"163.172.0.0/16",
"163.172.208.0/20",
"195.154.0.0/16",
"212.129.0.0/18",
"212.47.224.0/19",
"212.83.128.0/19",
"212.83.160.0/19",
"51.15.0.0/16",
"51.15.0.0/17",
"51.158.0.0/15",
"51.158.128.0/17",
"62.210.0.0/16",
"62.4.0.0/19",
}

func (a Scaleway) GetProvider() provider.Provider {
return provider.Scaleway
}

func (a Scaleway) GetIPRanges() []*IPRange {
log.Info("Using static Scaleway ip ranges")

ranges := make([]*IPRange, 0)
for _, cidr := range scalewayRanges {
network, cat := ParseCIDR(cidr)
ranges = append(ranges, &IPRange{
Network: network,
Cat: cat,
})
}
var ScalewayASN = "12876"

func (a Scaleway) GetIPRanges() []*IPRange {
log.Info("[Scaleway] - Using ranges from ASN list (AS%s)", ScalewayASN)
ranges := getRangesForAsn(ScalewayASN)
log.Info("[Scaleway] - Found %d ranges for AS%s", len(ranges), ScalewayASN)
return ranges
}
28 changes: 14 additions & 14 deletions internal/source/source_tencent.go
Original file line number Diff line number Diff line change
Expand Up @@ -7,26 +7,26 @@ import (

type Tencent struct{}

const tencentFileURL = "https://raw.githubusercontent.com/devanshbatham/ip2cloud/main/data/tencent.txt"

func (a Tencent) GetProvider() provider.Provider {
return provider.Tencent
}

func (a Tencent) GetIPRanges() []*IPRange {
log.Info("Using static Tencent ip ranges")
var TencentASNs = []string{
// Tencent Cloud
"132591",
// Tencent Global
"132203",
// Tencent-CN
"45090",
}

func (a Tencent) GetIPRanges() []*IPRange {
ranges := make([]*IPRange, 0)
tencentRanges, err := LoadTextURLToRange(tencentFileURL)
if err != nil {
log.Fatal("Failed to load text url to range for Tencent", err)
}
for _, cidr := range tencentRanges {
network, cat := ParseCIDR(cidr)
ranges = append(ranges, &IPRange{
Network: network,
Cat: cat,
})
for _, asn := range TencentASNs {
log.Info("[Tencent] - Using ranges from ASN list (AS%s)", asn)
_ranges := getRangesForAsn(asn)
ranges = append(ranges, _ranges...)
log.Info("[Tencent] - Found %d ranges for AS%s", len(ranges), asn)
}
return ranges
}
22 changes: 6 additions & 16 deletions internal/source/source_ucloud.go
Original file line number Diff line number Diff line change
Expand Up @@ -7,26 +7,16 @@ import (

type Ucloud struct{}

const ucloudFileURL = "https://raw.githubusercontent.com/devanshbatham/ip2cloud/main/data/ucloud.txt"

func (a Ucloud) GetProvider() provider.Provider {
return provider.Ucloud
}

func (a Ucloud) GetIPRanges() []*IPRange {
log.Info("Using static Ucloud ip ranges")
// UCLOUD INFORMATION TECHNOLOGY (HK) LIMITED
var UcloudASN = "135377"

ranges := make([]*IPRange, 0)
ucloudRanges, err := LoadTextURLToRange(ucloudFileURL)
if err != nil {
log.Fatal("Failed to load text url to range for ucloud", err)
}
for _, cidr := range ucloudRanges {
network, cat := ParseCIDR(cidr)
ranges = append(ranges, &IPRange{
Network: network,
Cat: cat,
})
}
func (a Ucloud) GetIPRanges() []*IPRange {
log.Info("[Ucloud] - Using ranges from ASN list (AS%s)", UcloudASN)
ranges := getRangesForAsn(UcloudASN)
log.Info("[Ucloud] - Found %d ranges for AS%s", len(ranges), UcloudASN)
return ranges
}
1 change: 1 addition & 0 deletions internal/source/source_vercel.go
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ func (a Vercel) GetProvider() provider.Provider {
}

// TODO: Get this range dynamically
// NOTE: this is pretty flaky, as vercel is just a wrapper of AWS. Maybe this provider should be removed
// Source: https://networksdb.io/ip-addresses-of/vercel-inc
const vercelRange = "76.76.21.0/24"

Expand Down
68 changes: 68 additions & 0 deletions internal/source/utils.go
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
package source

import (
"bufio"
"context"
"encoding/json"
"errors"
Expand All @@ -9,7 +10,10 @@ import (
"net"
"net/http"
"strings"
"sync"
"time"

"github.com/Escape-Technologies/cloudfinder/internal/log"
)

const defaultHTTPTimeout = 30 * time.Second
Expand Down Expand Up @@ -128,3 +132,67 @@ func isPrivateNetwork(n *net.IPNet) bool {
}
return false
}

/// BGP TOOLS

// ensure we fill the map only once
var bgpToolsMutex = sync.Mutex{}

// ASN -> CIDR
var bgpToolsAsnRanges map[string][]*IPRange
var bgpToolsTableURL = "https://bgp.tools/table.txt"

// Fetches https://bgp.tools/table.txt and parses it into the ASN -> CIDR map
func getRangesForAsn(asn string) []*IPRange {
bgpToolsMutex.Lock()
if bgpToolsAsnRanges == nil {
bgpToolsAsnRanges = make(map[string][]*IPRange)

// Fill the map
log.Info("Fetching AS infos from %s", bgpToolsTableURL)
req, _ := http.NewRequest(http.MethodGet, bgpToolsTableURL, nil) // nolint: noctx
// bgp tools requires a descriptive user agent in case the program gets out of control
req.Header.Add("user-agent", "https://github.com/Escape-Technologies/cloudfinder - [email protected]")
res, err := http.DefaultClient.Do(req)
if err != nil {
log.Fatal("Error getting bgp tools table", err)
}
if res.StatusCode != http.StatusOK {
err := fmt.Errorf("status code: %d", res.StatusCode)
log.Fatal("Error getting bgp tools table", err)
}

scanner := bufio.NewScanner(res.Body)
// Read lines
for scanner.Scan() {
line := scanner.Text()
// Line is formatted as "<CIDR> <ASN>"
x := strings.Split(line, " ")
if len(x) != 2 { // nolint: mnd
continue
}
cidr, asn := x[0], x[1]

n, cat := ParseCIDR(cidr)
// Skip private networks
if isPrivateNetwork(n) {
continue
}
// Fill map
if _, ok := bgpToolsAsnRanges[asn]; !ok {
bgpToolsAsnRanges[asn] = make([]*IPRange, 0)
}
bgpToolsAsnRanges[asn] = append(bgpToolsAsnRanges[asn], &IPRange{
Network: n,
Cat: cat,
})
}
res.Body.Close()
log.Info("Got %d AS infos", len(bgpToolsAsnRanges))
}
bgpToolsMutex.Unlock()
if val, ok := bgpToolsAsnRanges[asn]; ok {
return val
}
return []*IPRange{}
}
2 changes: 1 addition & 1 deletion internal/static/hash.txt
Original file line number Diff line number Diff line change
@@ -1 +1 @@
56bb287f7dbb22a8ba1ccad5e16fe86e8bd3632de2a056b37ed939cd3692ab7f
47c10bda97e1961c363626868c7aa32aace0cff5557877bbdcd0ea48a506804e
Binary file modified internal/static/ipv4.gob
Binary file not shown.
Binary file modified internal/static/ipv6.gob
Binary file not shown.
Loading

0 comments on commit e28dba3

Please sign in to comment.