chore: End of life notice

Signed-off-by: Julien Riou <julien@riou.xyz>

.gitignore

@@ -8,3 +8,4 @@ restockbot.pid
 ferret.log
 shop.fql
 *.bak
+venv

README.md

@@ -1,3 +1,8 @@
+# END OF LIFE NOTICE
+
+Due to lack of time to do the maintenance, this repository is now archived.
+Don't hesitate to fork it!
+
 # RestockBot
 
 Year 2020 has been quite hard for hardware supply. Graphics cards are out of stock everywhere. Nobody can grab the new generation (AMD RX 6000 series, NVIDIA GeForce RTX 3000 series). Even older generations are hard to find. `RestockBot` is a bot that crawl retailers websites and notify when a product is available.
@@ -157,6 +162,11 @@ Options:
     * `amazon_fulfilled`: include only products packaged by Amazon
     * `amazon_merchant`: include only products sold by Amazon
     * `affiliate_links`: generate affiliate links with the partner tag
+* `nvidia_fe` (optional)
+    * `locations`: list of NVIDIA stores (ex `["es", "fr", "it"]`)
+    * `gpus`: list of models (ex: `["RTX 3060 Ti", "RTX 3070"]`)
+    * `user_agent`: user agent to simulate a real web browser (ex: `Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:102.0) Gecko/20100101 Firefox/102.0`)
+    * `timeout`: maximum time before closing the request (optional)
 * `twitter` (optional):
     * `consumer_key`: API key of your Twitter application
     * `consumer_secret`: API secret of your Twitter application
@@ -198,6 +208,7 @@ docker run -it --name restockbot --rm --link chromium:chromium -v $(pwd):/root/
 There are two modes:
 * **default**: without special argument, the bot parses websites and manage its own database
 * **API**: using the `-api` argument, the bot starts the HTTP API to expose data from the database
+* **monitor**: using the `-monitor` (optionaly with `-monitor-warning-timeout` and `-monitor-critical-timeout` arguments), the bot checks for last execution times per shop to return a Nagios compatible output
 
 ## How to contribute
 

VERSION

@@ -1 +1 @@
-0.6.1
+0.7.0

config.go

@@ -13,6 +13,7 @@ type Config struct {
 	TelegramConfig `json:"telegram"`
 	APIConfig      `json:"api"`
 	AmazonConfig   `json:"amazon"`
+	NvidiaFEConfig `json:"nvidia_fe"`
 	URLs           []string     `json:"urls"`
 	IncludeRegex   string       `json:"include_regex"`
 	ExcludeRegex   string       `json:"exclude_regex"`
@@ -66,6 +67,14 @@ type AmazonConfig struct {
 	AffiliateLinks  bool `json:"affiliate_links"`
 }
 
+// NvidiaFEConfig to store NVIDIA Founders Edition configuration
+type NvidiaFEConfig struct {
+	Locations []string `json:"locations"`
+	GPUs      []string `json:"gpus"`
+	UserAgent string   `json:"user_agent"`
+	Timeout   int      `json:"timeout"`
+}
+
 // PriceRange to store rules to filter products with price outside of the range
 type PriceRange struct {
 	Model    string  `json:"model"`
@@ -118,6 +127,11 @@ func (c *Config) HasDatabase() bool {
 	return c.DatabaseConfig.Type != "" && c.DatabaseConfig.DSN != ""
 }
 
+// HasNvidiaFE returns true when NVIDIA FE has been configured
+func (c *Config) HasNvidiaFE() bool {
+	return len(c.NvidiaFEConfig.Locations) > 0 && len(c.NvidiaFEConfig.GPUs) > 0
+}
+
 // HasAmazon returns true when Amazon has been configured
 func (c *Config) HasAmazon() bool {
 	var hasKeys, hasSearches, hasMarketplaces bool

db.go

@@ -5,21 +5,26 @@ import (
 	"gorm.io/driver/postgres"
 	"gorm.io/driver/sqlite"
 	"gorm.io/gorm"
+	"gorm.io/gorm/logger"
 )
 
+var gconfig = &gorm.Config{
+	Logger: logger.Default.LogMode(logger.Silent),
+}
+
 // NewDatabaseFromConfig creates a database object from configuration structure
 func NewDatabaseFromConfig(config DatabaseConfig) (*gorm.DB, error) {
 	switch config.Type {
 	case "postgres":
-		return gorm.Open(postgres.New(postgres.Config{DSN: config.DSN}), &gorm.Config{})
+		return gorm.Open(postgres.New(postgres.Config{DSN: config.DSN}), gconfig)
 	case "mysql":
-		return gorm.Open(mysql.New(mysql.Config{DSN: config.DSN}), &gorm.Config{})
+		return gorm.Open(mysql.New(mysql.Config{DSN: config.DSN}), gconfig)
 	default:
-		return gorm.Open(sqlite.Open(config.DSN), &gorm.Config{})
+		return gorm.Open(sqlite.Open(config.DSN), gconfig)
 	}
 }
 
 // NewDatabaseFromFile creates a database object from path to file (passed as argument)
 func NewDatabaseFromFile(path string) (*gorm.DB, error) {
-	return gorm.Open(sqlite.Open(path), &gorm.Config{})
+	return gorm.Open(sqlite.Open(path), gconfig)
 }

main.go

@@ -53,6 +53,9 @@ func main() {
 	pidWaitTimeout := flag.Int("pid-wait-timeout", 0, "Seconds to wait before giving up when another instance is running")
 	retention := flag.Int("retention", 0, "Automatically remove products from the database with this number of days old (disabled by default)")
 	api := flag.Bool("api", false, "Start the HTTP API")
+	monitor := flag.Bool("monitor", false, "Perform health check with Nagios output")
+	warningTimeout := flag.Int("monitor-warning-timeout", 300, "Raise a warning alert when the last execution time has reached this number of seconds (see -monitor)")
+	criticalTimeout := flag.Int("monitor-critical-timeout", 600, "Raise a critical alert when the last execution time has reached this number of seconds (see -monitor)")
 
 	flag.Parse()
 
@@ -138,6 +141,11 @@ func main() {
 		}
 	}
 
+	// start monitoring
+	if *monitor {
+		os.Exit(Monitor(db, *warningTimeout, *criticalTimeout))
+	}
+
 	// start the api
 	if *api {
 		log.Fatal(StartAPI(db, config.APIConfig))
@@ -216,6 +224,20 @@ func main() {
 		}
 	}
 
+	if config.HasNvidiaFE() {
+		// create a parser for all locations
+		for _, location := range config.NvidiaFEConfig.Locations {
+			parser, err := NewNvidiaFRParser(location, config.NvidiaFEConfig.GPUs, config.NvidiaFEConfig.UserAgent, config.NvidiaFEConfig.Timeout)
+			if err != nil {
+				log.Warnf("could not create NVIDIA FE parser for location %s: %s", location, err)
+				continue
+			}
+
+			parsers = append(parsers, parser)
+			log.Debugf("parser %s registered", parser)
+		}
+	}
+
 	// parse asynchronously
 	var wg sync.WaitGroup
 	jobsCount := 0

monitoring.go

@@ -0,0 +1,128 @@
+package main
+
+import (
+	"fmt"
+	"strings"
+	"time"
+
+	log "github.com/sirupsen/logrus"
+	"gorm.io/gorm"
+)
+
+const (
+	// NagiosOk return the Nagios OK code (see https://nagios-plugins.org/doc/guidelines.html#AEN78)
+	NagiosOk = 0
+	// NagiosWarning return the Nagios WARNING code (see https://nagios-plugins.org/doc/guidelines.html#AEN78)
+	NagiosWarning = 1
+	// NagiosCritical return the Nagios CRITICAL code (see https://nagios-plugins.org/doc/guidelines.html#AEN78)
+	NagiosCritical = 2
+	// NagiosUnknown return the Nagios UNKNOWN code (see https://nagios-plugins.org/doc/guidelines.html#AEN78)
+	NagiosUnknown = 3
+)
+
+// MonitoringResult to store result of Nagios checks
+type MonitoringResult struct {
+	ShopName   string
+	Message    string
+	ReturnCode int
+}
+
+// String returns a string to print a MonitoringResult nicely
+func (m MonitoringResult) String() string {
+	return fmt.Sprintf("%s %s (rc = %d)", m.ShopName, m.Message, m.ReturnCode)
+}
+
+// ReturnCodeString returns a string to print a ReturnCode nicely
+func ReturnCodeString(rc int) string {
+	switch rc {
+	case NagiosOk:
+		return "OK"
+	case NagiosWarning:
+		return "WARN"
+	case NagiosCritical:
+		return "CRIT"
+	default:
+		return "UNK"
+	}
+}
+
+// FormatMonitoringResults to print a list of MonitoringResult nicely
+func FormatMonitoringResults(results []MonitoringResult) string {
+	var s []string
+	for _, result := range results {
+		s = append(s, fmt.Sprintf("%s %s", result.ShopName, result.Message))
+	}
+	return strings.Join(s, ", ")
+}
+
+// Monitor will check for last execution time for each shop and return either
+// a warning or critical alert when the threshold has been reached
+func Monitor(db *gorm.DB, warningTimeout int, criticalTimeout int) (rc int) {
+
+	// Find date and time thresholds
+	warningTime := time.Now().Add(-time.Duration(warningTimeout) * time.Second)
+	criticalTime := time.Now().Add(-time.Duration(criticalTimeout) * time.Second)
+
+	// Map to sort monitoring result by status code
+	resultMap := make(map[int][]MonitoringResult)
+
+	// List shops
+	var shops []Shop
+	trx := db.Find(&shops)
+	if trx.Error != nil {
+		fmt.Printf("%s\n", trx.Error)
+		return NagiosUnknown
+	}
+
+	for _, shop := range shops {
+
+		result := MonitoringResult{
+			ShopName:   shop.Name,
+			ReturnCode: NagiosOk,
+		}
+
+		// Fetch last execution time
+		var product Product
+		trx := db.Where(Product{ShopID: shop.ID}).Order("updated_at desc").First(&product)
+		if trx.Error == gorm.ErrRecordNotFound {
+			result.Message = "has not been updated"
+			result.ReturnCode = NagiosCritical
+			resultMap[NagiosCritical] = append(resultMap[result.ReturnCode], result)
+			continue
+		}
+		if trx.Error != nil {
+			fmt.Printf("%s\n", trx.Error)
+			return NagiosUnknown
+		}
+
+		// Compare to thresholds and add to result map
+		diff := int(time.Now().Sub(product.UpdatedAt.Local()).Seconds())
+		result.Message = fmt.Sprintf("updated %d seconds ago", diff)
+
+		if product.UpdatedAt.Before(criticalTime) {
+			result.ReturnCode = NagiosCritical
+		} else if product.UpdatedAt.Before(warningTime) {
+			result.ReturnCode = NagiosWarning
+		} else {
+		}
+		log.Info(result)
+		resultMap[result.ReturnCode] = append(resultMap[result.ReturnCode], result)
+	}
+
+	var message string
+
+	if len(resultMap[NagiosWarning]) > 0 {
+		rc = NagiosWarning
+		message = FormatMonitoringResults(resultMap[NagiosWarning])
+	} else if len(resultMap[NagiosCritical]) > 0 {
+		rc = NagiosCritical
+		message = FormatMonitoringResults(resultMap[NagiosCritical])
+	} else {
+		rc = NagiosOk
+		message = "All shops have been updated recently"
+	}
+
+	// Print output
+	fmt.Printf("%s - %s\n", ReturnCodeString(rc), message)
+	return
+}

parser_nvidia_fe.go

@@ -0,0 +1,197 @@
+package main
+
+import (
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"net/http"
+	"net/url"
+	"strconv"
+	"time"
+
+	log "github.com/sirupsen/logrus"
+)
+
+// Mapping between GPU model and NVIDIA SKU on the API
+var nvidiaSKUs = map[string]string{
+	"RTX 3060 Ti": "NVGFT060T",
+	"RTX 3070":    "NVGFT070",
+	"RTX 3070 Ti": "NVGFT070T",
+	"RTX 3080":    "NVGFT080",
+	"RTX 3080 Ti": "NVGFT080T",
+	"RTX 3090":    "NVGFT090",
+	"RTX 3090 Ti": "NVGFT090T",
+}
+
+// Mapping between location and currency
+var nvidiaCurrencies = map[string]string{
+	"es": "EUR",
+	"fr": "EUR",
+	"it": "EUR",
+}
+
+var supportedNvidiaGpus = []string{"RTX 3060 Ti", "RTX 3070", "RTX 3070 Ti", "RTX 3080", "RTX 3080 Ti", "RTX 3090", "RTX 3090 Ti"}
+var supportedNvidiaFELocations = []string{"es", "fr", "it"}
+
+// NvidiaFEParser to parse NVIDIA Founder Edition website parser
+// Implements the Parser interface
+type NvidiaFEParser struct {
+	location  string
+	gpus      []string
+	userAgent string
+	client    *http.Client
+}
+
+// NewNvidiaFRParser creates a parser for NVIDIA Founders Edition website for a specific location
+// Takes a location (ex: fr) and GPU (ex: RTX 3070, RTX 3090)
+func NewNvidiaFRParser(location string, gpus []string, userAgent string, timeout int) (*NvidiaFEParser, error) {
+	// Check supported locales
+	if !ContainsString(supportedNvidiaFELocations, location) {
+		return nil, fmt.Errorf("location %s not supported (expect one of %s", location, supportedNvidiaFELocations)
+	}
+
+	// Check supported GPU list
+	for _, gpu := range gpus {
+		if !ContainsString(supportedNvidiaGpus, gpu) {
+			return nil, fmt.Errorf("GPU %s not supported, expected one of %s", gpu, supportedNvidiaGpus)
+		}
+	}
+
+	// Empty user agent will return an error on the NVIDIA API
+	// It's probably a security measure to avoid third-party robots
+	if userAgent == "" {
+		return nil, fmt.Errorf("user agent required (use the same one as your web browser)")
+	}
+
+	client := &http.Client{Timeout: time.Duration(timeout) * time.Second}
+
+	return &NvidiaFEParser{
+		location:  location,
+		gpus:      gpus,
+		userAgent: userAgent,
+		client:    client,
+	}, nil
+}
+
+// ShopName returns a nice name for NVIDIA Founders Edition website
+// Implements the Parser interface
+func (p *NvidiaFEParser) ShopName() (string, error) {
+	return fmt.Sprintf("nvidia.com/%s-%s/shop", p.location, p.location), nil
+}
+
+// String to print NvidiaFEParser
+// Implements the Parser interface
+func (p *NvidiaFEParser) String() string {
+	return fmt.Sprintf("NvidiaFEParser<%s>", p.location)
+}
+
+// NvidiaFEResponse to store NVIDIA API response
+type NvidiaFEResponse struct {
+	Success bool `json:"success"`
+	ListMap []struct {
+		IsActive string `json:"is_active"`
+		Price    string `json:"price"`
+	}
+}
+
+// Parse NVIDIA store API to return list of products
+// Implements Parser interface
+func (p *NvidiaFEParser) Parse() ([]*Product, error) {
+	var products []*Product
+	for _, gpu := range p.gpus {
+		sku := nvidiaSKUs[gpu]
+		apiURL := fmt.Sprintf("https://api.store.nvidia.com/partner/v1/feinventory?status=1&skus=%s&locale=%s-%s", sku, p.location, p.location)
+
+		req, err := http.NewRequest(http.MethodGet, apiURL, nil)
+		if err != nil {
+			return nil, fmt.Errorf("failed to create new request: %s", err)
+		}
+
+		req.Header.Set("User-Agent", p.userAgent)
+		req.Header.Set("Accept", "application/json")
+
+		log.Debugf("requesting NVIDIA API: %s", req)
+
+		res, err := p.client.Do(req)
+		if err != nil {
+			return nil, fmt.Errorf("failed to request NVIDIA API: %s", err)
+		}
+
+		if res.Body != nil {
+			defer res.Body.Close()
+		}
+
+		body, err := ioutil.ReadAll(res.Body)
+		if err != nil {
+			return nil, fmt.Errorf("failed to read body: %s", err)
+		}
+
+		if res.StatusCode != http.StatusOK {
+			log.Debugf("%s", body)
+			return nil, fmt.Errorf("NVIDIA API returned %d", res.StatusCode)
+		}
+
+		response := NvidiaFEResponse{}
+		err = json.Unmarshal(body, &response)
+		if err != nil {
+			return nil, fmt.Errorf("failed to parse JSON response: %s", err)
+		}
+
+		if !response.Success {
+			return nil, fmt.Errorf("NVIDIA API returned an applicative failure for GPU %s and location %s", gpu, p.location)
+		}
+
+		for _, element := range response.ListMap {
+
+			var product = &Product{
+				Name:          gpu,
+				PriceCurrency: nvidiaCurrencies[p.location],
+			}
+
+			available, err := strconv.ParseBool(element.IsActive)
+			if err != nil {
+				return nil, fmt.Errorf("failed to parse bool from response: %s", err)
+			}
+			product.Available = available
+
+			productPrice, err := strconv.ParseFloat(element.Price, 64)
+			if err != nil {
+				return nil, fmt.Errorf("failed to parse float from response: %s", err)
+			}
+			product.Price = productPrice
+
+			productURL, err := createNvidiaFEProductURL(p.location, gpu)
+			if err != nil {
+				return nil, fmt.Errorf("failed to create product URL: %s", err)
+			}
+			product.URL = productURL
+			products = append(products, product)
+		}
+
+	}
+	return products, nil
+}
+
+// Create the product URL
+// Ex: https://store.nvidia.com/fr-fr/geforce/store/gpu/?page=1&limit=100&locale=fr-fr&category=GPU&gpu=RTX%203080&manufacturer=NVIDIA
+func createNvidiaFEProductURL(location string, gpu string) (string, error) {
+	locale := fmt.Sprintf("%s-%s", location, location)
+
+	productURL, err := url.Parse("https://store.nvidia.com")
+	if err != nil {
+		return "", err
+	}
+
+	productURL.Path += fmt.Sprintf("%s/geforce/store/gpu/", locale)
+
+	params := url.Values{}
+	params.Add("page", "1")
+	params.Add("limit", "100")
+	params.Add("locale", locale)
+	params.Add("category", "GPU")
+	params.Add("gpu", gpu)
+	params.Add("manufacturer", "NVIDIA")
+
+	productURL.RawQuery = params.Encode()
+	return productURL.String(), nil
+}