feat: add price range filter (#26)

To avoid scalpers' price, the bot now understand filters on prices
using a minimum and maximum value, in a currency and a pattern to
detect the model.

Example:
```
"price_ranges": [
  {"model": "3090", "max": 3000, "currency": "EUR"},
  {"model": "3080", "max": 1600, "currency": "EUR"},
  {"model": "3070", "max": 1200, "currency": "EUR"}
],
```

More details in README.md.

Signed-off-by: Julien Riou <julien@riou.xyz>

README.md

@@ -172,6 +172,7 @@ Options:
     * `enable_replies`: reply to original message when product is not available anymore
 * `include_regex` (optional): include products with a name matching this regexp
 * `exclude_regex` (optional): exclude products with a name matching this regexp
+* `price_ranges` (optional): define price ranges for products based on the model. List of rules containing `model` (regex to apply to the product name, string), `min` (minimum expected price, float), `max` (maximum expected price, float), `currency` (price currency used by the filter, string). For example `{"price_ranges":[{"model": "3090", "min": 0, "max": 3000, "currency": "EUR"}]}`
 * `browser_address` (optional): set headless browser address (ex: `http://127.0.0.1:9222`)
 * `api` (optional):
     * `address`: listen address for the REST API (ex: `127.0.0.1:8000`)

config.go

@@ -16,6 +16,7 @@ type Config struct {
 	URLs           []string     `json:"urls"`
 	IncludeRegex   string       `json:"include_regex"`
 	ExcludeRegex   string       `json:"exclude_regex"`
+	PriceRanges    []PriceRange `json:"price_ranges"`
 	BrowserAddress string       `json:"browser_address"`
 }
 
@@ -65,6 +66,14 @@ type AmazonConfig struct {
 	AffiliateLinks  bool `json:"affiliate_links"`
 }
 
+// PriceRange to store rules to filter products with price outside of the range
+type PriceRange struct {
+	Model    string  `json:"model"`
+	Min      float64 `json:"min"`
+	Max      float64 `json:"max"`
+	Currency string  `json:"currency"`
+}
+
 // NewConfig creates a Config struct
 func NewConfig() *Config {
 	return &Config{}

currency_converter.go

@@ -0,0 +1,96 @@
+package main
+
+import (
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"net/http"
+	"strings"
+
+	log "github.com/sirupsen/logrus"
+)
+
+// CurrencyConverter to cache rates of different currency pairs
+type CurrencyConverter struct {
+	rates map[string]float64
+}
+
+// NewCurrencyConverter to create a CurrencyConverter
+func NewCurrencyConverter() *CurrencyConverter {
+	return &CurrencyConverter{
+		rates: make(map[string]float64),
+	}
+}
+
+// Convert an amount in a given currency to another currency
+// Eventually fetch rate from a remote API then cache the result
+func (c *CurrencyConverter) Convert(amount float64, fromCurrency string, toCurrency string) (float64, error) {
+	var err error
+
+	if fromCurrency == toCurrency {
+		return amount, nil
+	}
+
+	// exclude invalid currencies
+	if fromCurrency == "" || toCurrency == "" {
+		return 0.0, fmt.Errorf("invalid currency pair used for convertion (from='%s', to='%s')", fromCurrency, toCurrency)
+	}
+
+	// searching currency pair rate in cache
+	pair := fmt.Sprintf("%s%s", strings.ToLower(fromCurrency), strings.ToLower(toCurrency))
+	rate, exists := c.rates[pair]
+	if !exists {
+		// fetching rate from api
+		rate, err = c.getRate(fromCurrency, toCurrency)
+		if err != nil {
+			return 0.0, err
+		}
+		// store rate in cache
+		c.rates[pair] = rate
+	}
+
+	return rate * amount, nil
+}
+
+// CurrencyResponse to unmarshall JSON response from API
+type CurrencyResponse struct {
+	Date string  `json:"date"`
+	Rate float64 `json:"rate"`
+}
+
+// getRate retreives rate from a remote API
+func (c *CurrencyConverter) getRate(fromCurrency string, toCurrency string) (float64, error) {
+	if fromCurrency == "" || toCurrency == "" {
+		return 0.0, fmt.Errorf("invalid currency pair used for convertion (from=%s, to=%s)", fromCurrency, toCurrency)
+	}
+
+	// lowering currency names to match the API route
+	fromCurrency = strings.ToLower(fromCurrency)
+	toCurrency = strings.ToLower(toCurrency)
+
+	log.Debugf("fetching %s%s rate from currency api", fromCurrency, toCurrency)
+	resp, err := http.Get("https://cdn.jsdelivr.net/gh/fawazahmed0/currency-api@1/latest/currencies/" + fromCurrency + "/" + toCurrency + ".json")
+	if err != nil {
+		return 0.0, fmt.Errorf("could not retreive currency rates for pair %s%s: %s", fromCurrency, toCurrency, err)
+	}
+	defer resp.Body.Close()
+
+	body, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		return 0.0, fmt.Errorf("could not parse currency rates response for pair %s%s: %s", fromCurrency, toCurrency, err)
+	}
+
+	// response has a dynamic name for the rate
+	//   -> {"date": "2021-05-22", "usd": 1.218125}
+	// making it predictable
+	//   -> {"date": "2021-05-22", "rate": 1.218125}}
+	bodyParsed := strings.Replace(string(body), toCurrency, "rate", 1)
+
+	var response CurrencyResponse
+	err = json.Unmarshal([]byte(bodyParsed), &response)
+	if err != nil {
+		return 0.0, err
+	}
+
+	return response.Rate, nil
+}

currency_converter_test.go

@@ -0,0 +1,44 @@
+package main
+
+import (
+	"fmt"
+	"testing"
+
+	"github.com/jarcoal/httpmock"
+)
+
+func TestCurrencyConverterConvert(t *testing.T) {
+	httpmock.Activate()
+	defer httpmock.DeactivateAndReset()
+
+	httpmock.RegisterResponder("GET", "https://cdn.jsdelivr.net/gh/fawazahmed0/currency-api@1/latest/currencies/eur/chf.json",
+		httpmock.NewStringResponder(200, `{"date": "2021-05-22", "chf": 1.093894}`))
+
+	tests := []struct {
+		amount      float64
+		source      string
+		destination string
+		expected    float64
+	}{
+		{1.0, "EUR", "EUR", 1.0},      // same currency (EUR/EUR)
+		{1.0, "EUR", "CHF", 1.093894}, // different currency (EUR/CHF)
+		{1.0, "EUR", "CHF", 1.093894}, // different currency (EUR/CHF) with cache
+	}
+
+	converter := NewCurrencyConverter()
+
+	for i, tc := range tests {
+		t.Run(fmt.Sprintf("TestCurrencyConverterConvert#%d", i), func(t *testing.T) {
+
+			converted, err := converter.Convert(tc.amount, tc.source, tc.destination)
+
+			if err != nil {
+				t.Errorf("could not convert %.2f from %s to %s: %s", tc.amount, tc.source, tc.destination, err)
+			} else if converted != tc.expected {
+				t.Errorf("to convert %.2f from %s to %s, got '%.2f', want '%.2f'", tc.amount, tc.source, tc.destination, converted, tc.expected)
+			} else {
+				t.Logf("to convert %.2f from %s to %s, got '%.2f', want '%.2f'", tc.amount, tc.source, tc.destination, converted, tc.expected)
+			}
+		})
+	}
+}

filter_range.go

@@ -6,15 +6,20 @@ import (
 	log "github.com/sirupsen/logrus"
 )
 
+// DefaultCurrency to fallback when no currency is provided
+const DefaultCurrency = "USD"
+
 // RangeFilter to store the pattern to match product model and price limits
 type RangeFilter struct {
 	model     *regexp.Regexp
 	min       float64
 	max       float64
+	currency  string
+	converter *CurrencyConverter
 }
 
 // NewRangeFilter to create a RangeFilter
-func NewRangeFilter(regex string, min float64, max float64) (*RangeFilter, error) {
+func NewRangeFilter(regex string, min float64, max float64, currency string, converter *CurrencyConverter) (*RangeFilter, error) {
 	var err error
 	var compiledRegex *regexp.Regexp
 
@@ -26,23 +31,56 @@ func NewRangeFilter(regex string, min float64, max float64) (*RangeFilter, error
 		}
 	}
 
+	var detectedCurrency string
+	if currency != "" {
+		detectedCurrency = currency
+	} else {
+		detectedCurrency = DefaultCurrency
+	}
+
 	return &RangeFilter{
 		model:     compiledRegex,
 		min:       min,
 		max:       max,
+		converter: converter,
+		currency:  detectedCurrency,
 	}, nil
 }
 
 // Include returns false when a product name matches the model regex and price is outside of the range
 // implements the Filter interface
 func (f *RangeFilter) Include(product *Product) bool {
+	// include products with a missing model regex
 	if f.model == nil {
+		log.Debugf("product %s included because range filter model is missing", product.Name)
 		return true
 	}
-	if f.model.MatchString(product.Name) && product.Price < f.min || product.Price > f.max {
+
-		log.Debugf("product %s excluded because price for the model is outside of the range [%.2f-%.2f]", product.Name, f.min, f.max)
+	// include products with a different model
+	if !f.model.MatchString(product.Name) {
+		log.Debugf("product %s included because range filter model hasn't been detected in the product name", product.Name)
+		return true
+	}
+
+	// convert price to the filter currency
+	convertedPrice, err := f.converter.Convert(product.Price, product.PriceCurrency, f.currency)
+	if err != nil {
+		log.Warnf("could not convert price %.2f %s to %s for range filter: %s", product.Price, product.PriceCurrency, f.currency, err)
+		return true
+	}
+
+	// include prices with unlimited maximum if min is respected
+	if f.max == 0 && convertedPrice > f.max && f.min <= convertedPrice {
+		log.Debugf("product %s included because max value is unlimited and converted price of %.2f%s is higher than lower limit of %.2f%s", product.Name, convertedPrice, f.currency, f.min, f.currency)
+		return true
+	}
+
+	// include prices inside the range
+	if f.min <= convertedPrice && convertedPrice <= f.max {
+		log.Debugf("product %s included because range filter model matches and converted price is inside of the range", product.Name)
+		return true
+	}
+
+	log.Debugf("product %s excluded because range filter model matches and converted price is outside of the range", product.Name)
 	return false
-	}
-	log.Debugf("product %s included because price range filter is not applicable", product.Name)
-	return true
 }

filter_range_test.go

@@ -7,36 +7,38 @@ import (
 
 func TestRangeFilter(t *testing.T) {
 	tests := []struct {
-		name     string  // product name
+		product  *Product
-		price    float64 // product price
 		model    string  // model regex to apply on the product name
 		min      float64 // minimum price
 		max      float64 // maximum price
+		currency string  // price currency
 		included bool    // should be included or not
 	}{
-		{"MSI GeForce RTX 3090 GAMING X", 99.99, "3090", 50.0, 100.0, true},   // model match and price is in the range, should be included
+		{&Product{Name: "MSI GeForce RTX 3090 GAMING X", Price: 99.99, PriceCurrency: "EUR"}, "3090", 50.0, 100.0, "EUR", true},   // model match and price is in the range, should be included
-		{"MSI GeForce RTX 3090 GAMING X", 99.99, "3080", 50.0, 100.0, true},   // model doesn't match, should be included
+		{&Product{Name: "MSI GeForce RTX 3090 GAMING X", Price: 99.99, PriceCurrency: "EUR"}, "3080", 50.0, 100.0, "EUR", true},   // model doesn't match, should be included
-		{"MSI GeForce RTX 3090 GAMING X", 999.99, "3090", 50.0, 100.0, false}, // model match and price is outside of the range, shoud not be included
+		{&Product{Name: "MSI GeForce RTX 3090 GAMING X", Price: 999.99, PriceCurrency: "EUR"}, "3090", 50.0, 100.0, "EUR", false}, // model match and price is outside of the range, shoud not be included
-		{"MSI GeForce RTX 3090 GAMING X", 99.99, "", 50.0, 100.0, true},       // model regex is missing, should be included
+		{&Product{Name: "MSI GeForce RTX 3090 GAMING X", Price: 99.99, PriceCurrency: "EUR"}, "", 50.0, 100.0, "EUR", true},       // model regex is missing, should be included
+		{&Product{Name: "MSI GeForce RTX 3090 GAMING X", Price: 99.99, PriceCurrency: "EUR"}, "3090", 50.0, 0.0, "EUR", true},     // upper limit is missing, should be included
 	}
 
+	converter := NewCurrencyConverter()
+
 	for i, tc := range tests {
 		t.Run(fmt.Sprintf("TestRangeFilter#%d", i), func(t *testing.T) {
-			product := &Product{Name: tc.name, Price: tc.price}
+			filter, err := NewRangeFilter(tc.model, tc.min, tc.max, tc.currency, converter)
-			filter, err := NewRangeFilter(tc.model, tc.min, tc.max)
 			if err != nil {
 				t.Errorf("cannot create filter with model regex '%s' and price range [%.2f, %.2f]: %s", tc.model, tc.min, tc.max, err)
 			}
 
-			included := filter.Include(product)
+			included := filter.Include(tc.product)
 
 			if included != tc.included {
-				t.Errorf("product '%s' with model regex '%s' and range [%.2f, %.2f]: got included=%t, want included=%t", tc.name, tc.model, tc.min, tc.max, included, tc.included)
+				t.Errorf("product '%s' of price %.2f%s with model regex '%s' and range [%.2f, %.2f]: got included=%t, want included=%t", tc.product.Name, tc.product.Price, tc.product.PriceCurrency, tc.model, tc.min, tc.max, included, tc.included)
 			} else {
 				if included {
-					t.Logf("product '%s' included by model regex '%s' and range [%.2f, %.2f]", tc.name, tc.model, tc.min, tc.max)
+					t.Logf("product '%s' included by model regex '%s' and range [%.2f, %.2f]", tc.product.Name, tc.model, tc.min, tc.max)
 				} else {
-					t.Logf("product '%s' excluded by model regex '%s' and range [%.2f, %.2f]", tc.name, tc.model, tc.min, tc.max)
+					t.Logf("product '%s' excluded by model regex '%s' and range [%.2f, %.2f]", tc.product.Name, tc.model, tc.min, tc.max)
 				}
 			}
 

go.mod

@@ -8,6 +8,7 @@ require (
 	github.com/dghubble/oauth1 v0.7.0
 	github.com/go-telegram-bot-api/telegram-bot-api/v5 v5.0.0-rc1
 	github.com/gorilla/mux v1.8.0
+	github.com/jarcoal/httpmock v1.0.8 // indirect
 	github.com/sirupsen/logrus v1.8.0
 	github.com/spiegel-im-spiegel/pa-api v0.9.0
 	gorm.io/driver/mysql v1.0.5

go.sum

@@ -115,6 +115,8 @@ github.com/jackc/puddle v0.0.0-20190608224051-11cab39313c9/go.mod h1:m4B5Dj62Y0f
 github.com/jackc/puddle v1.1.0/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk=
 github.com/jackc/puddle v1.1.1/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk=
 github.com/jackc/puddle v1.1.3/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk=
+github.com/jarcoal/httpmock v1.0.8 h1:8kI16SoO6LQKgPE7PvQuV+YuD/inwHd7fOOe2zMbo4k=
+github.com/jarcoal/httpmock v1.0.8/go.mod h1:ATjnClrvW/3tijVmpL/va5Z3aAyGvqU3gCT8nX0Txik=
 github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E=
 github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc=
 github.com/jinzhu/now v1.1.1 h1:g39TucaRWyV3dwDO++eEc6qf8TVIQ/Da48WmqjZ3i7E=

main.go

@@ -179,6 +179,16 @@ func main() {
 		}
 		filters = append(filters, excludeFilter)
 	}
+	if len(config.PriceRanges) > 0 {
+		converter := NewCurrencyConverter()
+		for _, pr := range config.PriceRanges {
+			rangeFilter, err := NewRangeFilter(pr.Model, pr.Min, pr.Max, pr.Currency, converter)
+			if err != nil {
+				log.Fatalf("cannot create price range filter: %s", err)
+			}
+			filters = append(filters, rangeFilter)
+		}
+	}
 
 	// create parsers
 	parsers := []Parser{}