Add global config flag to enable splitting queries

Adds a global config flag to enable splitting queries that would hit the rollup table, but the rollup table has a delay SLA configured. In that case, this feature allows splitting a query into to; one that gets the data from the rollups table until the time where it's guaranteed to be available, and the rest from the raw table.
OpenTSDB · johann8384 · Feb 24, 2021 · Oct 16, 2019 · Oct 16, 2019 · Oct 28, 2019
commit cdbc9d0b4a95df756a6ec3dae13d8b7eb39450c1
diff --git a/src/core/TSDB.java b/src/core/TSDB.java
@@ -183,7 +183,13 @@ public enum OperationMode {
 
   /** Whether or not to block writing of derived rollups/pre-ags */
   private final boolean rollups_block_derived;
-
+
+  /**
+   * Whether or not to enable splitting rollup queries if the rollup table is lagging
+   * Global config setting: tsd.rollups.split_query.enable = true
+   */
+  private final boolean rollups_split_queries;
+
   /** An optional histogram manger used when the TSD will be dealing with
    * histograms and sketches. Instantiated ONLY if 
    * {@link #initializePlugins(boolean)} was called.*/
@@ -312,13 +318,15 @@ public TSDB(final HBaseClient client, final Config config) {
       agg_tag_key = config.getString("tsd.rollups.agg_tag_key");
       raw_agg_tag_value = config.getString("tsd.rollups.raw_agg_tag_value");
       rollups_block_derived = config.getBoolean("tsd.rollups.block_derived");
+      rollups_split_queries = config.getBoolean("tsd.rollups.split_query.enable");
     } else {
       rollup_config = null;
       default_interval = null;
       tag_raw_data = false;
       agg_tag_key = null;
       raw_agg_tag_value = null;
       rollups_block_derived = false;
+      rollups_split_queries = false;
     }
 
     QueryStats.setEnableDuplicates(
@@ -549,7 +557,7 @@ public void initializePlugins(final boolean init_rpcs) {
           uid_filter.getClass().getCanonicalName() + "] version: "
           + uid_filter.version());
     }
-    
+
     // finally load the histo manager after plugins have been loaded.
     if (config.hasProperty("tsd.core.histograms.config")) {
       histogram_manager = new HistogramCodecManager(this);
@@ -566,7 +574,7 @@ public void initializePlugins(final boolean init_rpcs) {
   public final Authentication getAuth() {
     return this.authentication;
   }
-
+  
   /** 
    * Returns the configured HBase client
    * @return The HBase client
@@ -2124,7 +2132,18 @@ public String getRawTagValue() {
     return raw_agg_tag_value;
   }
 
-  /** @return The optional histogram manager registered to this TSD. 
+  /**
+   * Returns whether the global config setting allows splitting rollups queries
+   * if the rollups table to be hit is lagging.
+   *
+   * @return Whether or not splitting rollup queries is enabled
+   * @since 2.4
+   */
+  public boolean isRollupsSplittingEnabled() {
+    return rollups_split_queries;
+  }
+
+  /** @return The optional histogram manager registered to this TSD.
    * @since 2.4 */
   public HistogramCodecManager histogramManager() {
     return histogram_manager;

diff --git a/src/utils/Config.java b/src/utils/Config.java
@@ -577,6 +577,7 @@ protected void setDefaults() {
     default_map.put("tsd.rollups.agg_tag_key", "_aggregate");
     default_map.put("tsd.rollups.raw_agg_tag_value", "RAW");
     default_map.put("tsd.rollups.block_derived", "true");
+    default_map.put("tsd.rollups.split_query.enable", "false");
     default_map.put("tsd.rtpublisher.enable", "false");
     default_map.put("tsd.rtpublisher.plugin", "");
     default_map.put("tsd.search.enable", "false");

diff --git a/test/core/TestTSDBAddAggregatePoint.java b/test/core/TestTSDBAddAggregatePoint.java
@@ -84,6 +84,7 @@ public void beforeLocal() throws Exception {
     Whitebox.setInternalState(tsdb, "default_interval", 
         rollup_config.getRollupInterval("1m"));
     Whitebox.setInternalState(tsdb, "rollups_block_derived", true);
+    Whitebox.setInternalState(tsdb, "rollups_split_queries", false);
     Whitebox.setInternalState(tsdb, "agg_tag_key", 
         config.getString("tsd.rollups.agg_tag_key"));
     Whitebox.setInternalState(tsdb, "raw_agg_tag_value", 

diff --git a/test/core/TestTSDBAddAggregatePointSalted.java b/test/core/TestTSDBAddAggregatePointSalted.java
@@ -85,6 +85,7 @@ public void beforeLocal() throws Exception {
     Whitebox.setInternalState(tsdb, "default_interval", 
         rollup_config.getRollupInterval("1m"));
     Whitebox.setInternalState(tsdb, "rollups_block_derived", true);
+    Whitebox.setInternalState(tsdb, "rollups_split_queries", false);
     Whitebox.setInternalState(tsdb, "agg_tag_key", 
         config.getString("tsd.rollups.agg_tag_key"));
     Whitebox.setInternalState(tsdb, "raw_agg_tag_value", 

diff --git a/test/tsd/TestRollupRpc.java b/test/tsd/TestRollupRpc.java
@@ -100,7 +100,8 @@ public void beforeLocal() throws Exception {
     storage.addTable("tsdb-rollup-agg-1h".getBytes(), families);
     storage.addTable("tsdb-agg".getBytes(), families);
     Whitebox.setInternalState(tsdb, "rollups_block_derived", true);
-    Whitebox.setInternalState(tsdb, "agg_tag_key", 
+    Whitebox.setInternalState(tsdb, "rollups_split_queries", false);
+    Whitebox.setInternalState(tsdb, "agg_tag_key",
         config.getString("tsd.rollups.agg_tag_key"));
     Whitebox.setInternalState(tsdb, "raw_agg_tag_value", 
         config.getString("tsd.rollups.raw_agg_tag_value"));
@@ -850,5 +851,4 @@ public void httpUnknownInterval() throws Exception {
     validateCounters(0, 1, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0);
     validateSEH(false);
   }
-
-}
+}