Alerts should not wake up idle clickhouse service

[k-anshul]

Running alert queries every 10 minutes for tables created externally can prevent the clickhouse cloud service from becoming idle.

1. Check the service status before checking for alerts.
2. Do not trigger alert queries if service is idle.
3. Optionally have a instance var that checks whether to wake up service to check alerts or not.
4. Consider increasing this to 20 minutes since the default time for service idle check is 15 minutes.

Here is a exact vs best-effort implementation of checking service status. The exact implementation require more user inputs. The best-effort implementation is rudimentary and will only work when both connection max time and alert check time is less than service idle time.

// SacaledToZero implements OLAPStore.
// TODO :: clickhouse APIs have very strict rate limits, 10 requests per 10 second per API key
// Consider caching this result if we are going to add more use cases around this.
func (c *connection) ScaledToZero() bool {
	if c.config.APIKeyID == "" {
		// no api key provided resort to a rudimentary solution
		return c.scaleToZeroBasic()
	}
	req, err := http.NewRequest(http.MethodGet, fmt.Sprintf("https://api.clickhouse.cloud/v1/organizations/%s/services/%s", c.config.OrganizationID, c.config.ServiceID), http.NoBody)
	if err != nil {
		c.logger.Warn("failed to create clickhouse cloud API request", zap.Error(err))
		return c.scaleToZeroBasic()
	}
	req.SetBasicAuth(c.config.APIKeyID, c.config.APIKeySecret)

	resp, err := c.cloudAPI.Do(req)
	if err != nil {
		c.logger.Warn("failed to get clickhouse cloud API response", zap.Error(err))
		return c.scaleToZeroBasic()
	}
	defer resp.Body.Close()

	if resp.StatusCode != http.StatusOK {
		c.logger.Warn("failed to get clickhouse cloud API response", zap.Int("status_code", resp.StatusCode))
		return c.scaleToZeroBasic()
	}

	// parse response
	var response struct {
		Result struct {
			State string `json:"state"`
		} `json:"result"`
	}
	err = json.NewDecoder(resp.Body).Decode(&response)
	if err != nil {
		c.logger.Warn("failed to decode clickhouse cloud API response", zap.Error(err))
		return c.scaleToZeroBasic()
	}
	return strings.EqualFold(response.Result.State, "idle")
}

func (c *connection) scaleToZeroBasic() bool {
	stats := c.db.DB.Stats()
	return stats.OpenConnections == 0
}