feat: add price range filter (#26)
To avoid scalpers' price, the bot now understand filters on prices
using a minimum and maximum value, in a currency and a pattern to
detect the model.
Example:
```
"price_ranges": [
{"model": "3090", "max": 3000, "currency": "EUR"},
{"model": "3080", "max": 1600, "currency": "EUR"},
{"model": "3070", "max": 1200, "currency": "EUR"}
],
```
More details in README.md.
Signed-off-by: Julien Riou <julien@riou.xyz>
This commit is contained in:
parent
ba791435b7
commit
da532104f8
9 changed files with 231 additions and 28 deletions
|
|
@ -172,6 +172,7 @@ Options:
|
|||
* `enable_replies`: reply to original message when product is not available anymore
|
||||
* `include_regex` (optional): include products with a name matching this regexp
|
||||
* `exclude_regex` (optional): exclude products with a name matching this regexp
|
||||
* `price_ranges` (optional): define price ranges for products based on the model. List of rules containing `model` (regex to apply to the product name, string), `min` (minimum expected price, float), `max` (maximum expected price, float), `currency` (price currency used by the filter, string). For example `{"price_ranges":[{"model": "3090", "min": 0, "max": 3000, "currency": "EUR"}]}`
|
||||
* `browser_address` (optional): set headless browser address (ex: `http://127.0.0.1:9222`)
|
||||
* `api` (optional):
|
||||
* `address`: listen address for the REST API (ex: `127.0.0.1:8000`)
|
||||
|
|
|
|||
17
config.go
17
config.go
|
|
@ -13,10 +13,11 @@ type Config struct {
|
|||
TelegramConfig `json:"telegram"`
|
||||
APIConfig `json:"api"`
|
||||
AmazonConfig `json:"amazon"`
|
||||
URLs []string `json:"urls"`
|
||||
IncludeRegex string `json:"include_regex"`
|
||||
ExcludeRegex string `json:"exclude_regex"`
|
||||
BrowserAddress string `json:"browser_address"`
|
||||
URLs []string `json:"urls"`
|
||||
IncludeRegex string `json:"include_regex"`
|
||||
ExcludeRegex string `json:"exclude_regex"`
|
||||
PriceRanges []PriceRange `json:"price_ranges"`
|
||||
BrowserAddress string `json:"browser_address"`
|
||||
}
|
||||
|
||||
// DatabaseConfig to store database configuration
|
||||
|
|
@ -65,6 +66,14 @@ type AmazonConfig struct {
|
|||
AffiliateLinks bool `json:"affiliate_links"`
|
||||
}
|
||||
|
||||
// PriceRange to store rules to filter products with price outside of the range
|
||||
type PriceRange struct {
|
||||
Model string `json:"model"`
|
||||
Min float64 `json:"min"`
|
||||
Max float64 `json:"max"`
|
||||
Currency string `json:"currency"`
|
||||
}
|
||||
|
||||
// NewConfig creates a Config struct
|
||||
func NewConfig() *Config {
|
||||
return &Config{}
|
||||
|
|
|
|||
96
currency_converter.go
Normal file
96
currency_converter.go
Normal file
|
|
@ -0,0 +1,96 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"net/http"
|
||||
"strings"
|
||||
|
||||
log "github.com/sirupsen/logrus"
|
||||
)
|
||||
|
||||
// CurrencyConverter to cache rates of different currency pairs
|
||||
type CurrencyConverter struct {
|
||||
rates map[string]float64
|
||||
}
|
||||
|
||||
// NewCurrencyConverter to create a CurrencyConverter
|
||||
func NewCurrencyConverter() *CurrencyConverter {
|
||||
return &CurrencyConverter{
|
||||
rates: make(map[string]float64),
|
||||
}
|
||||
}
|
||||
|
||||
// Convert an amount in a given currency to another currency
|
||||
// Eventually fetch rate from a remote API then cache the result
|
||||
func (c *CurrencyConverter) Convert(amount float64, fromCurrency string, toCurrency string) (float64, error) {
|
||||
var err error
|
||||
|
||||
if fromCurrency == toCurrency {
|
||||
return amount, nil
|
||||
}
|
||||
|
||||
// exclude invalid currencies
|
||||
if fromCurrency == "" || toCurrency == "" {
|
||||
return 0.0, fmt.Errorf("invalid currency pair used for convertion (from='%s', to='%s')", fromCurrency, toCurrency)
|
||||
}
|
||||
|
||||
// searching currency pair rate in cache
|
||||
pair := fmt.Sprintf("%s%s", strings.ToLower(fromCurrency), strings.ToLower(toCurrency))
|
||||
rate, exists := c.rates[pair]
|
||||
if !exists {
|
||||
// fetching rate from api
|
||||
rate, err = c.getRate(fromCurrency, toCurrency)
|
||||
if err != nil {
|
||||
return 0.0, err
|
||||
}
|
||||
// store rate in cache
|
||||
c.rates[pair] = rate
|
||||
}
|
||||
|
||||
return rate * amount, nil
|
||||
}
|
||||
|
||||
// CurrencyResponse to unmarshall JSON response from API
|
||||
type CurrencyResponse struct {
|
||||
Date string `json:"date"`
|
||||
Rate float64 `json:"rate"`
|
||||
}
|
||||
|
||||
// getRate retreives rate from a remote API
|
||||
func (c *CurrencyConverter) getRate(fromCurrency string, toCurrency string) (float64, error) {
|
||||
if fromCurrency == "" || toCurrency == "" {
|
||||
return 0.0, fmt.Errorf("invalid currency pair used for convertion (from=%s, to=%s)", fromCurrency, toCurrency)
|
||||
}
|
||||
|
||||
// lowering currency names to match the API route
|
||||
fromCurrency = strings.ToLower(fromCurrency)
|
||||
toCurrency = strings.ToLower(toCurrency)
|
||||
|
||||
log.Debugf("fetching %s%s rate from currency api", fromCurrency, toCurrency)
|
||||
resp, err := http.Get("https://cdn.jsdelivr.net/gh/fawazahmed0/currency-api@1/latest/currencies/" + fromCurrency + "/" + toCurrency + ".json")
|
||||
if err != nil {
|
||||
return 0.0, fmt.Errorf("could not retreive currency rates for pair %s%s: %s", fromCurrency, toCurrency, err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
body, err := ioutil.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
return 0.0, fmt.Errorf("could not parse currency rates response for pair %s%s: %s", fromCurrency, toCurrency, err)
|
||||
}
|
||||
|
||||
// response has a dynamic name for the rate
|
||||
// -> {"date": "2021-05-22", "usd": 1.218125}
|
||||
// making it predictable
|
||||
// -> {"date": "2021-05-22", "rate": 1.218125}}
|
||||
bodyParsed := strings.Replace(string(body), toCurrency, "rate", 1)
|
||||
|
||||
var response CurrencyResponse
|
||||
err = json.Unmarshal([]byte(bodyParsed), &response)
|
||||
if err != nil {
|
||||
return 0.0, err
|
||||
}
|
||||
|
||||
return response.Rate, nil
|
||||
}
|
||||
44
currency_converter_test.go
Normal file
44
currency_converter_test.go
Normal file
|
|
@ -0,0 +1,44 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"testing"
|
||||
|
||||
"github.com/jarcoal/httpmock"
|
||||
)
|
||||
|
||||
func TestCurrencyConverterConvert(t *testing.T) {
|
||||
httpmock.Activate()
|
||||
defer httpmock.DeactivateAndReset()
|
||||
|
||||
httpmock.RegisterResponder("GET", "https://cdn.jsdelivr.net/gh/fawazahmed0/currency-api@1/latest/currencies/eur/chf.json",
|
||||
httpmock.NewStringResponder(200, `{"date": "2021-05-22", "chf": 1.093894}`))
|
||||
|
||||
tests := []struct {
|
||||
amount float64
|
||||
source string
|
||||
destination string
|
||||
expected float64
|
||||
}{
|
||||
{1.0, "EUR", "EUR", 1.0}, // same currency (EUR/EUR)
|
||||
{1.0, "EUR", "CHF", 1.093894}, // different currency (EUR/CHF)
|
||||
{1.0, "EUR", "CHF", 1.093894}, // different currency (EUR/CHF) with cache
|
||||
}
|
||||
|
||||
converter := NewCurrencyConverter()
|
||||
|
||||
for i, tc := range tests {
|
||||
t.Run(fmt.Sprintf("TestCurrencyConverterConvert#%d", i), func(t *testing.T) {
|
||||
|
||||
converted, err := converter.Convert(tc.amount, tc.source, tc.destination)
|
||||
|
||||
if err != nil {
|
||||
t.Errorf("could not convert %.2f from %s to %s: %s", tc.amount, tc.source, tc.destination, err)
|
||||
} else if converted != tc.expected {
|
||||
t.Errorf("to convert %.2f from %s to %s, got '%.2f', want '%.2f'", tc.amount, tc.source, tc.destination, converted, tc.expected)
|
||||
} else {
|
||||
t.Logf("to convert %.2f from %s to %s, got '%.2f', want '%.2f'", tc.amount, tc.source, tc.destination, converted, tc.expected)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
|
@ -6,15 +6,20 @@ import (
|
|||
log "github.com/sirupsen/logrus"
|
||||
)
|
||||
|
||||
// DefaultCurrency to fallback when no currency is provided
|
||||
const DefaultCurrency = "USD"
|
||||
|
||||
// RangeFilter to store the pattern to match product model and price limits
|
||||
type RangeFilter struct {
|
||||
model *regexp.Regexp
|
||||
min float64
|
||||
max float64
|
||||
model *regexp.Regexp
|
||||
min float64
|
||||
max float64
|
||||
currency string
|
||||
converter *CurrencyConverter
|
||||
}
|
||||
|
||||
// NewRangeFilter to create a RangeFilter
|
||||
func NewRangeFilter(regex string, min float64, max float64) (*RangeFilter, error) {
|
||||
func NewRangeFilter(regex string, min float64, max float64, currency string, converter *CurrencyConverter) (*RangeFilter, error) {
|
||||
var err error
|
||||
var compiledRegex *regexp.Regexp
|
||||
|
||||
|
|
@ -26,23 +31,56 @@ func NewRangeFilter(regex string, min float64, max float64) (*RangeFilter, error
|
|||
}
|
||||
}
|
||||
|
||||
var detectedCurrency string
|
||||
if currency != "" {
|
||||
detectedCurrency = currency
|
||||
} else {
|
||||
detectedCurrency = DefaultCurrency
|
||||
}
|
||||
|
||||
return &RangeFilter{
|
||||
model: compiledRegex,
|
||||
min: min,
|
||||
max: max,
|
||||
model: compiledRegex,
|
||||
min: min,
|
||||
max: max,
|
||||
converter: converter,
|
||||
currency: detectedCurrency,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// Include returns false when a product name matches the model regex and price is outside of the range
|
||||
// implements the Filter interface
|
||||
func (f *RangeFilter) Include(product *Product) bool {
|
||||
// include products with a missing model regex
|
||||
if f.model == nil {
|
||||
log.Debugf("product %s included because range filter model is missing", product.Name)
|
||||
return true
|
||||
}
|
||||
if f.model.MatchString(product.Name) && product.Price < f.min || product.Price > f.max {
|
||||
log.Debugf("product %s excluded because price for the model is outside of the range [%.2f-%.2f]", product.Name, f.min, f.max)
|
||||
return false
|
||||
|
||||
// include products with a different model
|
||||
if !f.model.MatchString(product.Name) {
|
||||
log.Debugf("product %s included because range filter model hasn't been detected in the product name", product.Name)
|
||||
return true
|
||||
}
|
||||
log.Debugf("product %s included because price range filter is not applicable", product.Name)
|
||||
return true
|
||||
|
||||
// convert price to the filter currency
|
||||
convertedPrice, err := f.converter.Convert(product.Price, product.PriceCurrency, f.currency)
|
||||
if err != nil {
|
||||
log.Warnf("could not convert price %.2f %s to %s for range filter: %s", product.Price, product.PriceCurrency, f.currency, err)
|
||||
return true
|
||||
}
|
||||
|
||||
// include prices with unlimited maximum if min is respected
|
||||
if f.max == 0 && convertedPrice > f.max && f.min <= convertedPrice {
|
||||
log.Debugf("product %s included because max value is unlimited and converted price of %.2f%s is higher than lower limit of %.2f%s", product.Name, convertedPrice, f.currency, f.min, f.currency)
|
||||
return true
|
||||
}
|
||||
|
||||
// include prices inside the range
|
||||
if f.min <= convertedPrice && convertedPrice <= f.max {
|
||||
log.Debugf("product %s included because range filter model matches and converted price is inside of the range", product.Name)
|
||||
return true
|
||||
}
|
||||
|
||||
log.Debugf("product %s excluded because range filter model matches and converted price is outside of the range", product.Name)
|
||||
return false
|
||||
}
|
||||
|
|
|
|||
|
|
@ -7,36 +7,38 @@ import (
|
|||
|
||||
func TestRangeFilter(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string // product name
|
||||
price float64 // product price
|
||||
product *Product
|
||||
model string // model regex to apply on the product name
|
||||
min float64 // minimum price
|
||||
max float64 // maximum price
|
||||
currency string // price currency
|
||||
included bool // should be included or not
|
||||
}{
|
||||
{"MSI GeForce RTX 3090 GAMING X", 99.99, "3090", 50.0, 100.0, true}, // model match and price is in the range, should be included
|
||||
{"MSI GeForce RTX 3090 GAMING X", 99.99, "3080", 50.0, 100.0, true}, // model doesn't match, should be included
|
||||
{"MSI GeForce RTX 3090 GAMING X", 999.99, "3090", 50.0, 100.0, false}, // model match and price is outside of the range, shoud not be included
|
||||
{"MSI GeForce RTX 3090 GAMING X", 99.99, "", 50.0, 100.0, true}, // model regex is missing, should be included
|
||||
{&Product{Name: "MSI GeForce RTX 3090 GAMING X", Price: 99.99, PriceCurrency: "EUR"}, "3090", 50.0, 100.0, "EUR", true}, // model match and price is in the range, should be included
|
||||
{&Product{Name: "MSI GeForce RTX 3090 GAMING X", Price: 99.99, PriceCurrency: "EUR"}, "3080", 50.0, 100.0, "EUR", true}, // model doesn't match, should be included
|
||||
{&Product{Name: "MSI GeForce RTX 3090 GAMING X", Price: 999.99, PriceCurrency: "EUR"}, "3090", 50.0, 100.0, "EUR", false}, // model match and price is outside of the range, shoud not be included
|
||||
{&Product{Name: "MSI GeForce RTX 3090 GAMING X", Price: 99.99, PriceCurrency: "EUR"}, "", 50.0, 100.0, "EUR", true}, // model regex is missing, should be included
|
||||
{&Product{Name: "MSI GeForce RTX 3090 GAMING X", Price: 99.99, PriceCurrency: "EUR"}, "3090", 50.0, 0.0, "EUR", true}, // upper limit is missing, should be included
|
||||
}
|
||||
|
||||
converter := NewCurrencyConverter()
|
||||
|
||||
for i, tc := range tests {
|
||||
t.Run(fmt.Sprintf("TestRangeFilter#%d", i), func(t *testing.T) {
|
||||
product := &Product{Name: tc.name, Price: tc.price}
|
||||
filter, err := NewRangeFilter(tc.model, tc.min, tc.max)
|
||||
filter, err := NewRangeFilter(tc.model, tc.min, tc.max, tc.currency, converter)
|
||||
if err != nil {
|
||||
t.Errorf("cannot create filter with model regex '%s' and price range [%.2f, %.2f]: %s", tc.model, tc.min, tc.max, err)
|
||||
}
|
||||
|
||||
included := filter.Include(product)
|
||||
included := filter.Include(tc.product)
|
||||
|
||||
if included != tc.included {
|
||||
t.Errorf("product '%s' with model regex '%s' and range [%.2f, %.2f]: got included=%t, want included=%t", tc.name, tc.model, tc.min, tc.max, included, tc.included)
|
||||
t.Errorf("product '%s' of price %.2f%s with model regex '%s' and range [%.2f, %.2f]: got included=%t, want included=%t", tc.product.Name, tc.product.Price, tc.product.PriceCurrency, tc.model, tc.min, tc.max, included, tc.included)
|
||||
} else {
|
||||
if included {
|
||||
t.Logf("product '%s' included by model regex '%s' and range [%.2f, %.2f]", tc.name, tc.model, tc.min, tc.max)
|
||||
t.Logf("product '%s' included by model regex '%s' and range [%.2f, %.2f]", tc.product.Name, tc.model, tc.min, tc.max)
|
||||
} else {
|
||||
t.Logf("product '%s' excluded by model regex '%s' and range [%.2f, %.2f]", tc.name, tc.model, tc.min, tc.max)
|
||||
t.Logf("product '%s' excluded by model regex '%s' and range [%.2f, %.2f]", tc.product.Name, tc.model, tc.min, tc.max)
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
1
go.mod
1
go.mod
|
|
@ -8,6 +8,7 @@ require (
|
|||
github.com/dghubble/oauth1 v0.7.0
|
||||
github.com/go-telegram-bot-api/telegram-bot-api/v5 v5.0.0-rc1
|
||||
github.com/gorilla/mux v1.8.0
|
||||
github.com/jarcoal/httpmock v1.0.8 // indirect
|
||||
github.com/sirupsen/logrus v1.8.0
|
||||
github.com/spiegel-im-spiegel/pa-api v0.9.0
|
||||
gorm.io/driver/mysql v1.0.5
|
||||
|
|
|
|||
2
go.sum
2
go.sum
|
|
@ -115,6 +115,8 @@ github.com/jackc/puddle v0.0.0-20190608224051-11cab39313c9/go.mod h1:m4B5Dj62Y0f
|
|||
github.com/jackc/puddle v1.1.0/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk=
|
||||
github.com/jackc/puddle v1.1.1/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk=
|
||||
github.com/jackc/puddle v1.1.3/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk=
|
||||
github.com/jarcoal/httpmock v1.0.8 h1:8kI16SoO6LQKgPE7PvQuV+YuD/inwHd7fOOe2zMbo4k=
|
||||
github.com/jarcoal/httpmock v1.0.8/go.mod h1:ATjnClrvW/3tijVmpL/va5Z3aAyGvqU3gCT8nX0Txik=
|
||||
github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E=
|
||||
github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc=
|
||||
github.com/jinzhu/now v1.1.1 h1:g39TucaRWyV3dwDO++eEc6qf8TVIQ/Da48WmqjZ3i7E=
|
||||
|
|
|
|||
10
main.go
10
main.go
|
|
@ -179,6 +179,16 @@ func main() {
|
|||
}
|
||||
filters = append(filters, excludeFilter)
|
||||
}
|
||||
if len(config.PriceRanges) > 0 {
|
||||
converter := NewCurrencyConverter()
|
||||
for _, pr := range config.PriceRanges {
|
||||
rangeFilter, err := NewRangeFilter(pr.Model, pr.Min, pr.Max, pr.Currency, converter)
|
||||
if err != nil {
|
||||
log.Fatalf("cannot create price range filter: %s", err)
|
||||
}
|
||||
filters = append(filters, rangeFilter)
|
||||
}
|
||||
}
|
||||
|
||||
// create parsers
|
||||
parsers := []Parser{}
|
||||
|
|
|
|||
Reference in a new issue