Added: Poor man's load balancing

theotherp · Feb 27, 2017 · d9baa08 · d9baa08
1 parent 205375c
commit d9baa08
Show file tree

Hide file tree

Showing 6 changed files with 73 additions and 9 deletions.
diff --git a/changelog.md b/changelog.md
@@ -2,6 +2,10 @@
 
 ----------
 ### 0.2.208
+Added: Poor man's load balancing. For each indexer you can define a number x and for every API call that indexer will be picked with a chance of 1/x. For example with a value of 3 it will on average be picked on every third API call (if it meets all other restrictions).
+ This should allow to distribute access to indexers with a low API hit limit over a day. It's up to you to find sensible values.  
+
+
 Changed: Rewrote the caps check error handling some more because this causes problems and questions very often. Added a message saying that you can set the IDs manually and which links to the Wiki. 
 
 Fixed: Database error when calling --help. See [#563](https://github.com/theotherp/nzbhydra/issues/563).

diff --git a/nzbhydra/config.py b/nzbhydra/config.py
@@ -119,7 +119,7 @@
     "main": {
         "apikey": ''.join(random.choice('0123456789ABCDEF') for i in range(32)),
         "branch": "master",
-        "configVersion": 37,
+        "configVersion": 38,
         "dereferer": "http://www.dereferer.org/?$s",
         "debug": False,
         "externalUrl": None,
@@ -765,6 +765,12 @@ def migrateConfig(config):
                     if "showIndexerSelection" not in user.keys():
                         user["showIndexerSelection"] = True
 
+        if config["main"]["configVersion"] == 37:
+            with version_update(config, 38):
+                addLogMessage(20, "Disabling load limiting for indexers by default")
+                for indexer in config["indexers"]:
+                    indexer["loadLimitOnRandom"] = None
+
     return config
 
 

diff --git a/nzbhydra/search.py b/nzbhydra/search.py
@@ -7,6 +7,7 @@
 import datetime
 import hashlib
 import logging
+import random
 import re
 from itertools import groupby
 from sets import Set
@@ -128,7 +129,7 @@ def checkHitOrDownloadLimit(p):
                 logger.info("Did not pick %s because its API hit limit of %d was reached. Will pick again after %02d:00" % (p, p.settings.hitLimit, p.settings.hitLimitResetTime))
             else:
                 try:
-                    firstHitTimeInWindow = arrow.get(list(apiHitsQuery.order_by(IndexerApiAccess.time.desc()).offset(p.settings.hitLimit-1).dicts())[0]["time"]).to("local")
+                    firstHitTimeInWindow = arrow.get(list(apiHitsQuery.order_by(IndexerApiAccess.time.desc()).offset(p.settings.hitLimit - 1).dicts())[0]["time"]).to("local")
                     nextHitAfter = arrow.get(firstHitTimeInWindow + datetime.timedelta(days=1))
                     logger.info("Did not pick %s because its API hit limit of %d was reached. Next possible hit at %s" % (p, p.settings.hitLimit, nextHitAfter.format('YYYY-MM-DD HH:mm')))
                 except IndexerApiAccess.DoesNotExist:
@@ -145,7 +146,7 @@ def checkHitOrDownloadLimit(p):
                 logger.info("Did not pick %s because its download limit of %d was reached. Will pick again after %02d:00" % (p, p.settings.downloadLimit, p.settings.hitLimitResetTime))
             else:
                 try:
-                    firstHitTimeInWindow = arrow.get(list(downloadsQuery.order_by(IndexerApiAccess.time.desc()).offset(p.settings.downloadLimit-1).limit(1).dicts())[0]["time"]).to("local")
+                    firstHitTimeInWindow = arrow.get(list(downloadsQuery.order_by(IndexerApiAccess.time.desc()).offset(p.settings.downloadLimit - 1).limit(1).dicts())[0]["time"]).to("local")
                     nextHitAfter = arrow.get(firstHitTimeInWindow + datetime.timedelta(days=1))
                     logger.info("Did not pick %s because its download limit of %d was reached. Next possible hit at %s" % (p, p.settings.downloadLimit, nextHitAfter.format('YYYY-MM-DD HH:mm')))
                 except IndexerApiAccess.DoesNotExist:
@@ -210,6 +211,11 @@ def pick_indexers(search_request):
             add_not_picked_indexer(notPickedReasons, "Query needed", p.name)
             continue
 
+        if p.settings.loadLimitOnRandom and not search_request.internal and not random.randint(1, p.settings.loadLimitOnRandom) == 1:
+            logger.debug("Did not pick %s because load limiting prevented it. Chances of this indexer being picked: %d/%d" % (p, 1, p.settings.loadLimitOnRandom))
+            add_not_picked_indexer(notPickedReasons, "Load limiting", p.name)
+            continue
+
         # If we can theoretically do that we must try to actually get the title, otherwise the indexer won't be able to search
         allow_query_generation = (config.InternalExternalSelection.internal in config.settings.searching.generate_queries and search_request.internal) or (config.InternalExternalSelection.external in config.settings.searching.generate_queries and not search_request.internal)
         if search_request.identifier_key is not None and not canUseIdKey(p, search_request.identifier_key):
@@ -342,7 +348,7 @@ def search(search_request):
         elif search_request.loadAll:
             logger.debug("All results requested. Continuing to search.")
         logger.debug("%d indexers still have results" % len(indexers_to_call))
-        
+
         if cache_entry["usedFallback"]:
             search_request.offset = cache_entry["offsetFallback"]
         else:
@@ -453,23 +459,23 @@ def search(search_request):
                 if len(cache_entry["results"]) == 0:
                     logger.info("No results found using ID based search. Getting title from ID to fall back")
                     call_fallback = True
-                                            
+
             if call_fallback:
                 title = infos.convertId(search_request.identifier_key, "title", search_request.identifier_value)
                 if title:
                     logger.info("Repeating search with title based query as fallback")
                     logger.info("Fallback title: %s" % (title))
-                                        
+
                     # Add title and remove identifier key/value from search
                     search_request.title = title
                     search_request.identifier_key = None
                     search_request.identifier_value = None
-                    
+
                     if config.settings.searching.idFallbackToTitlePerIndexer:
                         indexers_to_call = indexers_to_call_fallback
                     else:
                         indexers_to_call = [indexer for indexer, _ in cache_entry["indexer_infos"].items()]
-                    
+
                     cache_entry["usedFallback"] = True
                 else:
                     logger.info("Unable to find title for ID")

diff --git a/static/js/nzbhydra.js b/static/js/nzbhydra.js
diff --git a/static/js/nzbhydra.js.map b/static/js/nzbhydra.js.map
diff --git a/ui-src/js/config-fields-service.js b/ui-src/js/config-fields-service.js
@@ -897,6 +897,7 @@ function ConfigFields($injector) {
                             enabled: true,
                             categories: [],
                             downloadLimit: null,
+                            loadLimitOnRandom: null,
                             host: null,
                             apikey: null,
                             hitLimit: null,
@@ -1249,6 +1250,7 @@ function getIndexerPresets(configuredIndexers) {
                 hitLimit: null,
                 hitLimitResetTime: null,
                 host: "https://anizb.org",
+                loadLimitOnRandom: null,
                 name: "anizb",
                 password: null,
                 preselect: true,
@@ -1268,6 +1270,7 @@ function getIndexerPresets(configuredIndexers) {
                 hitLimit: null,
                 hitLimitResetTime: null,
                 host: "https://binsearch.info",
+                loadLimitOnRandom: null,
                 name: "Binsearch",
                 password: null,
                 preselect: true,
@@ -1287,6 +1290,7 @@ function getIndexerPresets(configuredIndexers) {
                 hitLimit: null,
                 hitLimitResetTime: null,
                 host: "https://www.nzbclub.com",
+                loadLimitOnRandom: null,
                 name: "NZBClub",
                 password: null,
                 preselect: true,
@@ -1308,6 +1312,7 @@ function getIndexerPresets(configuredIndexers) {
                 hitLimit: null,
                 hitLimitResetTime: null,
                 host: "https://nzbindex.com",
+                loadLimitOnRandom: null,
                 name: "NZBIndex",
                 password: null,
                 preselect: true,
@@ -1450,6 +1455,25 @@ function getIndexerBoxFields(model, parentModel, isInitial, injector) {
             }
         );
         fieldset.push(
+            {
+                key: 'loadLimitOnRandom',
+                type: 'horizontalInput',
+                templateOptions: {
+                    type: 'number',
+                    label: 'Load limiting',
+                    help: 'If set indexer will only be picked for one out of x API searches (on average)'
+                },
+                validators: {
+                    greaterThanZero: {
+                        expression: function ($viewValue, $modelValue) {
+                            var value = $modelValue || $viewValue;
+                            return angular.isUndefined(value) || value === null || value === "" || value > 1;
+                        },
+                        message: '"Value must be greater than 1"'
+                    }
+
+                }
+            },
             {
                 key: 'hitLimitResetTime',
                 type: 'horizontalInput',