diff --git a/README.md b/README.md index 5306dae..e033722 100644 --- a/README.md +++ b/README.md @@ -157,6 +157,11 @@ Options: * `amazon_fulfilled`: include only products packaged by Amazon * `amazon_merchant`: include only products sold by Amazon * `affiliate_links`: generate affiliate links with the partner tag +* `nvidia_fe` (optional) + * `locations`: list of NVIDIA stores (ex `["es", "fr", "it"]`) + * `gpus`: list of models (ex: `["RTX 3060 Ti", "RTX 3070"]`) + * `user_agent`: user agent to simulate a real web browser (ex: `Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:102.0) Gecko/20100101 Firefox/102.0`) + * `timeout`: maximum time before closing the request (optional) * `twitter` (optional): * `consumer_key`: API key of your Twitter application * `consumer_secret`: API secret of your Twitter application diff --git a/config.go b/config.go index d71468f..7c9c279 100644 --- a/config.go +++ b/config.go @@ -13,6 +13,7 @@ type Config struct { TelegramConfig `json:"telegram"` APIConfig `json:"api"` AmazonConfig `json:"amazon"` + NvidiaFEConfig `json:"nvidia_fe"` URLs []string `json:"urls"` IncludeRegex string `json:"include_regex"` ExcludeRegex string `json:"exclude_regex"` @@ -66,6 +67,14 @@ type AmazonConfig struct { AffiliateLinks bool `json:"affiliate_links"` } +// NvidiaFEConfig to store NVIDIA Founders Edition configuration +type NvidiaFEConfig struct { + Locations []string `json:"locations"` + GPUs []string `json:"gpus"` + UserAgent string `json:"user_agent"` + Timeout int `json:"timeout"` +} + // PriceRange to store rules to filter products with price outside of the range type PriceRange struct { Model string `json:"model"` @@ -118,6 +127,11 @@ func (c *Config) HasDatabase() bool { return c.DatabaseConfig.Type != "" && c.DatabaseConfig.DSN != "" } +// HasNvidiaFE returns true when NVIDIA FE has been configured +func (c *Config) HasNvidiaFE() bool { + return len(c.NvidiaFEConfig.Locations) > 0 && len(c.NvidiaFEConfig.GPUs) > 0 +} + // HasAmazon returns true when Amazon has been configured func (c *Config) HasAmazon() bool { var hasKeys, hasSearches, hasMarketplaces bool diff --git a/main.go b/main.go index 0709b2d..162696e 100644 --- a/main.go +++ b/main.go @@ -216,6 +216,20 @@ func main() { } } + if config.HasNvidiaFE() { + // create a parser for all locations + for _, location := range config.NvidiaFEConfig.Locations { + parser, err := NewNvidiaFRParser(location, config.NvidiaFEConfig.GPUs, config.NvidiaFEConfig.UserAgent, config.NvidiaFEConfig.Timeout) + if err != nil { + log.Warnf("could not create NVIDIA FE parser for location %s: %s", location, err) + continue + } + + parsers = append(parsers, parser) + log.Debugf("parser %s registered", parser) + } + } + // parse asynchronously var wg sync.WaitGroup jobsCount := 0 diff --git a/parser_nvidia_fe.go b/parser_nvidia_fe.go new file mode 100644 index 0000000..df0034c --- /dev/null +++ b/parser_nvidia_fe.go @@ -0,0 +1,195 @@ +package main + +import ( + "encoding/json" + "fmt" + "io/ioutil" + "net/http" + "net/url" + "strconv" + "time" + + log "github.com/sirupsen/logrus" +) + +// Mapping between GPU model and NVIDIA SKU on the API +var nvidiaSKUs = map[string]string{ + "RTX 3060 Ti": "NVGFT060T", + "RTX 3070": "NVGFT070", + "RTX 3070 Ti": "NVGFT070T", + "RTX 3080": "NVGFT080", + "RTX 3080 Ti": "NVGFT080T", + "RTX 3090": "NVGFT090", + "RTX 3090 Ti": "NVGFT090T", +} + +// Mapping between location and currency +var nvidiaCurrencies = map[string]string{ + "es": "EUR", + "fr": "EUR", + "it": "EUR", +} + +var supportedNvidiaGpus = []string{"RTX 3060 Ti", "RTX 3070", "RTX 3070 Ti", "RTX 3080", "RTX 3080 Ti", "RTX 3090", "RTX 3090 Ti"} +var supportedNvidiaFELocations = []string{"es", "fr", "it"} + +type NvidiaFEParser struct { + location string + gpus []string + userAgent string + client *http.Client +} + +// NewNvidiaFRParser creates a parser for NVIDIA Founders Edition website for a specific location +// Takes a location (ex: fr) and GPU (ex: RTX 3070, RTX 3090) +func NewNvidiaFRParser(location string, gpus []string, userAgent string, timeout int) (*NvidiaFEParser, error) { + // Check supported locales + if !ContainsString(supportedNvidiaFELocations, location) { + return nil, fmt.Errorf("location %s not supported (expect one of %s", location, supportedNvidiaFELocations) + } + + // Check supported GPU list + for _, gpu := range gpus { + if !ContainsString(supportedNvidiaGpus, gpu) { + return nil, fmt.Errorf("GPU %s not supported, expected one of %s", gpu, supportedNvidiaGpus) + } + } + + // Empty user agent will return an error on the NVIDIA API + // It's probably a security measure to avoid third-party robots + if userAgent == "" { + return nil, fmt.Errorf("user agent required (use the same one as your web browser)") + } + + client := &http.Client{Timeout: time.Duration(timeout) * time.Second} + + return &NvidiaFEParser{ + location: location, + gpus: gpus, + userAgent: userAgent, + client: client, + }, nil +} + +// ShopName returns a nice name for NVIDIA Founders Edition website +// Implements the Parser interface +func (p *NvidiaFEParser) ShopName() (string, error) { + return fmt.Sprintf("www.nvidia.com/%s-%s/shop/", p.location, p.location), nil +} + +// String to print NvidiaFEParser +// Implements the Parser interface +func (p *NvidiaFEParser) String() string { + return fmt.Sprintf("NvidiaFEParser<%s>", p.location) +} + +// NvidiaFEResponse to store NVIDIA API response +type NvidiaFEResponse struct { + Success bool `json:"success"` + ListMap []struct { + IsActive string `json:"is_active"` + Price string `json:"price"` + } +} + +// Parse NVIDIA store API to return list of products +// Implements Parser interface +func (p *NvidiaFEParser) Parse() ([]*Product, error) { + var products []*Product + for _, gpu := range p.gpus { + sku := nvidiaSKUs[gpu] + apiURL := fmt.Sprintf("https://api.store.nvidia.com/partner/v1/feinventory?status=1&skus=%s&locale=%s-%s", sku, p.location, p.location) + + req, err := http.NewRequest(http.MethodGet, apiURL, nil) + if err != nil { + return nil, fmt.Errorf("failed to create new request: %s", err) + } + + req.Header.Set("User-Agent", p.userAgent) + req.Header.Set("Accept", "application/json") + + log.Debugf("requesting NVIDIA API: %s", req) + + res, err := p.client.Do(req) + if err != nil { + return nil, fmt.Errorf("failed to request NVIDIA API: %s", err) + } + + if res.Body != nil { + defer res.Body.Close() + } + + body, err := ioutil.ReadAll(res.Body) + if err != nil { + return nil, fmt.Errorf("failed to read body: %s", err) + } + + if res.StatusCode != http.StatusOK { + log.Debugf("%s", body) + return nil, fmt.Errorf("NVIDIA API returned %d", res.StatusCode) + } + + response := NvidiaFEResponse{} + err = json.Unmarshal(body, &response) + if err != nil { + return nil, fmt.Errorf("failed to parse JSON response: %s", err) + } + + if !response.Success { + return nil, fmt.Errorf("NVIDIA API returned an applicative failure for GPU %s and location %s", gpu, p.location) + } + + for _, element := range response.ListMap { + + var product = &Product{ + Name: gpu, + PriceCurrency: nvidiaCurrencies[p.location], + } + + available, err := strconv.ParseBool(element.IsActive) + if err != nil { + return nil, fmt.Errorf("failed to parse bool from response: %s", err) + } + product.Available = available + + productPrice, err := strconv.ParseFloat(element.Price, 64) + if err != nil { + return nil, fmt.Errorf("failed to parse float from response: %s", err) + } + product.Price = productPrice + + productURL, err := createNvidiaFEProductURL(p.location, gpu) + if err != nil { + return nil, fmt.Errorf("failed to create product URL: %s", err) + } + product.URL = productURL + products = append(products, product) + } + + } + return products, nil +} + +// Create the product URL +// Ex: https://store.nvidia.com/fr-fr/geforce/store/gpu/?page=1&limit=100&locale=fr-fr&category=GPU&gpu=RTX%203080&manufacturer=NVIDIA +func createNvidiaFEProductURL(location string, gpu string) (string, error) { + locale := fmt.Sprintf("%s-%s", location, location) + + productURL, err := url.Parse("https://store.nvidia.com") + if err != nil { + return "", err + } + + productURL.Path += fmt.Sprintf("%s/geforce/store/gpu/", locale) + + params := url.Values{} + params.Add("page", "1") + params.Add("limit", "100") + params.Add("locale", locale) + params.Add("category", "GPU") + params.Add("gpu", gpu) + params.Add("manufacturer", "NVIDIA") + + productURL.RawQuery = params.Encode() + return productURL.String(), nil +}