controller: added request retry on 500

decort/controller.go

@@ -27,12 +27,14 @@ package decort
 import (
 	"bytes"
 	"crypto/tls"
+	"errors"
 	"fmt"
 	"io/ioutil"
 	"net/http"
 	"net/url"
 	"strconv"
 	"strings"
+	"time"
 
 	// "time"
 
@@ -378,28 +380,31 @@ func (config *ControllerCfg) decortAPICall(method string, api_name string, url_v
 		req.Header.Set("Authorization", fmt.Sprintf("bearer %s", config.jwt))
 	}
 
+	for i := 0; i < 5; i++ {
 		resp, err := config.cc_client.Do(req)
 		if err != nil {
 			return "", err
 		}
-	defer resp.Body.Close()
 
 		body, err := ioutil.ReadAll(resp.Body)
 		if err != nil {
 			return "", err
 		}
+		resp.Body.Close()
 		log.Debugf("decortAPICall: %s %s\n %s", method, api_name, body)
 
 		if resp.StatusCode == http.StatusOK {
 			return string(body), nil
 		} else {
+			if resp.StatusCode == http.StatusInternalServerError {
+				log.Warnf("got 500, retrying %d/5", i+1)
+				time.Sleep(time.Second * 5)
+				continue
+			}
 			return "", fmt.Errorf("decortAPICall: unexpected status code %d when calling API %q with request Body %q. Respone:\n%s",
 				resp.StatusCode, req.URL, params_str, body)
 		}
+	}
 
-	/*
+	return "", errors.New("number of retries exceeded")
-			if resp.StatusCode == StatusServiceUnavailable {
-		        return nil, fmt.Errorf("decortAPICall method called for incompatible authorization mode %q.", config.auth_mode_txt)
-			}
-	*/
 }