Add HTTP API

Add `-api` mode to start the HTTP API with the following routes:
- /health
- /shops
- /shops/:id
- /products
- /products/:id

Signed-off-by: Julien Riou <julien@riou.xyz>

README.md

@@ -122,6 +122,10 @@ Options:
 * `include_regex` (optional): include products with a name matching this regexp
 * `exclude_regex` (optional): exclude products with a name matching this regexp
 * `browser_address` (optional): set headless browser address (ex: `http://127.0.0.1:9222`)
+* `api` (optional):
+    * `address`: listen address for the REST API (ex: `127.0.0.1:8000`)
+    * `cert_file` (optional): use SSL and use this certificate file
+    * `key_file` (optional): use SSL and use this key file
 
 ## Usage
 
@@ -137,6 +141,12 @@ restockbot -help
 docker run -it --name restockbot --rm --link chromium:chromium -v $(pwd):/root/ restockbot:$(cat VERSION) restockbot -help
 ```
 
+## Execution modes
+
+There are two modes:
+* **default**: without special argument, the bot parses websites and manage its own database
+* **API**: using the `-api` argument, the bot starts the HTTP API to expose data from the database
+
 ## How to contribute
 
 Lint the code with pre-commit:

api.go

@@ -0,0 +1,170 @@
+package main
+
+import (
+	"fmt"
+	"net/http"
+	"strconv"
+	"time"
+
+	"encoding/json"
+
+	"github.com/gorilla/mux"
+	log "github.com/sirupsen/logrus"
+	"gorm.io/gorm"
+)
+
+// StatusWriter to log response status code
+type StatusWriter struct {
+	http.ResponseWriter
+	Status int
+}
+
+// NewStatusResponseWriter to create a new StatusWriter handler for HTTP logging
+func NewStatusResponseWriter(w http.ResponseWriter) *StatusWriter {
+	return &StatusWriter{
+		ResponseWriter: w,
+		Status:         http.StatusOK,
+	}
+}
+
+// WriteHeader to write status code to response
+func (sw *StatusWriter) WriteHeader(code int) {
+	sw.Status = code
+	sw.ResponseWriter.WriteHeader(code)
+}
+
+// LoggingMiddleware to log HTTP requests
+func LoggingMiddleware(r *mux.Router) mux.MiddlewareFunc {
+	return func(next http.Handler) http.Handler {
+		return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
+			start := time.Now()
+			sw := NewStatusResponseWriter(w)
+
+			defer func() {
+				log.Printf("%s %s %v %d %s %s", req.RemoteAddr, req.Method, time.Since(start), sw.Status, req.URL.Path, req.URL.RawQuery)
+			}()
+			next.ServeHTTP(w, req)
+		})
+	}
+}
+
+// handle health checks
+func handleHealth(w http.ResponseWriter, r *http.Request) {
+	fmt.Fprintf(w, "OK")
+}
+
+// shopsHandler to expose shops over HTTP with a database connection
+type shopsHandler struct {
+	db *gorm.DB
+}
+
+// ServeHTTP to implement the handle interface for serving shops
+func (h *shopsHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	var shops []Shop
+	trx := h.db.Find(&shops)
+	if trx.Error == nil {
+		w.WriteHeader(http.StatusOK)
+		w.Header().Set("Content-Type", "application/json")
+		json.NewEncoder(w).Encode(shops)
+	} else {
+		w.WriteHeader(http.StatusInternalServerError)
+	}
+}
+
+// shopHandler to expose shops over HTTP with a database connection
+type shopHandler struct {
+	db *gorm.DB
+}
+
+// ServeHTTP to implement the handle interface for serving shop
+func (h *shopHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	w.Header().Set("Content-Type", "application/json")
+	vars := mux.Vars(r)
+	id := vars["id"]
+
+	var shop Shop
+	trx := h.db.First(&shop, id)
+	if trx.Error == nil {
+		w.WriteHeader(http.StatusOK)
+		json.NewEncoder(w).Encode(shop)
+	} else {
+		w.WriteHeader(http.StatusInternalServerError)
+	}
+}
+
+// productsHandler to expose products over HTTP with a database connection
+type productsHandler struct {
+	db *gorm.DB
+}
+
+// ServeHTTP to implement the handle interface for serving products
+func (h *productsHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	w.Header().Set("Content-Type", "application/json")
+	var products []Product
+	var trx *gorm.DB
+	availableFilter := r.URL.Query().Get("available")
+
+	if availableFilter != "" {
+		available, err := strconv.ParseBool(availableFilter)
+		if err != nil {
+			log.Warnf("cannot parse available query to boolean: %s", err)
+			w.WriteHeader(http.StatusInternalServerError)
+			return
+		} else {
+			trx = h.db.Preload("Shop").Where(map[string]interface{}{"available": available}).Find(&products)
+		}
+	} else {
+		trx = h.db.Preload("Shop").Find(&products)
+	}
+
+	if trx.Error == nil {
+		w.WriteHeader(http.StatusOK)
+		json.NewEncoder(w).Encode(products)
+	} else {
+		w.WriteHeader(http.StatusInternalServerError)
+	}
+}
+
+// productHandler to expose product over HTTP with a database connection
+type productHandler struct {
+	db *gorm.DB
+}
+
+// ServeHTTP to implement the handle interface for serving product
+func (h *productHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	vars := mux.Vars(r)
+	id := vars["id"]
+
+	var product Product
+	trx := h.db.Preload("Shop").First(&product, id)
+	if trx.Error == nil {
+		w.WriteHeader(http.StatusOK)
+		w.Header().Set("Content-Type", "application/json")
+		json.NewEncoder(w).Encode(product)
+	} else {
+		w.WriteHeader(http.StatusInternalServerError)
+	}
+}
+
+// StartAPI to handle HTTP requests
+func StartAPI(db *gorm.DB, config ApiConfig) error {
+	router := mux.NewRouter().StrictSlash(true)
+
+	router.Path("/health").HandlerFunc(handleHealth)
+
+	router.Path("/shops").Handler(&shopsHandler{db: db})
+	router.Path("/shops/{id:[0-9]+}").Handler(&shopHandler{db: db})
+
+	router.Path("/products").Handler(&productsHandler{db: db})
+	router.Path("/products/{id:[0-9]+}").Handler(&productHandler{db: db})
+
+	// register middlewares
+	router.Use(LoggingMiddleware(router))
+
+	log.Printf("starting API on %s", config.Address)
+	if config.Certfile != "" && config.Keyfile != "" {
+		return http.ListenAndServeTLS(config.Address, config.Certfile, config.Keyfile, router)
+	} else {
+		return http.ListenAndServe(config.Address, router)
+	}
+}

config.go

@@ -10,6 +10,7 @@ import (
 type Config struct {
 	TwitterConfig  `json:"twitter"`
 	TelegramConfig `json:"telegram"`
+	ApiConfig      `json:"api"`
 	URLs           []string `json:"urls"`
 	IncludeRegex   string   `json:"include_regex"`
 	ExcludeRegex   string   `json:"exclude_regex"`
@@ -32,6 +33,13 @@ type TelegramConfig struct {
 	ChannelName string `json:"channel_name"`
 }
 
+// ApiConfig to store HTTP API configuration
+type ApiConfig struct {
+	Address  string `json:"address"`
+	Certfile string `json:"cert_file"`
+	Keyfile  string `json:"key_file"`
+}
+
 // NewConfig creates a Config struct
 func NewConfig() *Config {
 	return &Config{}

go.mod

@@ -7,6 +7,7 @@ require (
 	github.com/dghubble/go-twitter v0.0.0-20201011215211-4b180d0cc78d
 	github.com/dghubble/oauth1 v0.7.0
 	github.com/go-telegram-bot-api/telegram-bot-api/v5 v5.0.0-rc1
+	github.com/gorilla/mux v1.8.0
 	github.com/sirupsen/logrus v1.8.0
 	gorm.io/driver/sqlite v1.1.4
 	gorm.io/gorm v1.20.12

go.sum

@@ -49,6 +49,8 @@ github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGa
 github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
 github.com/gorilla/css v1.0.0 h1:BQqNyPTi50JCFMTw/b67hByjMVXZRwGha6wxVGkeihY=
 github.com/gorilla/css v1.0.0/go.mod h1:Dn721qIggHpt4+EFCcTLTU/vk5ySda2ReITrtgBl60c=
+github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI=
+github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So=
 github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc=
 github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
 github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E=

main.go

@@ -52,6 +52,7 @@ func main() {
 	workers := flag.Int("workers", 1, "number of workers for parsing shops")
 	pidFile := flag.String("pid-file", "", "write process ID to this file to disable concurrent executions")
 	pidWaitTimeout := flag.Int("pid-wait-timeout", 0, "seconds to wait before giving up when another instance is running")
+	api := flag.Bool("api", false, "Start the HTTP API")
 
 	flag.Parse()
 
@@ -114,6 +115,11 @@ func main() {
 		log.Fatalf("cannot create shops table")
 	}
 
+	// start the api
+	if *api {
+		log.Fatal(StartAPI(db, config.ApiConfig))
+	}
+
 	// register notifiers
 	notifiers := []Notifier{}
 

models.go

@@ -12,8 +12,8 @@ type Product struct {
 	Price         float64 `gorm:"not null" json:"price"`
 	PriceCurrency string  `gorm:"not null" json:"price_currency"`
 	Available     bool    `gorm:"not null;default:false" json:"available"`
-	ShopID        uint
-	Shop          Shop
+	ShopID        uint    `json:"shop_id"`
+	Shop          Shop    `json:"shop"`
 }
 
 // Equal compares a database product to another product
@@ -41,5 +41,5 @@ func (p *Product) ToMerge(o *Product) bool {
 // Shop represents a retailer website
 type Shop struct {
 	ID   uint   `gorm:"primaryKey"`
-	Name string `gorm:"unique"`
+	Name string `gorm:"unique" json:"name"`
 }