From 5a80f13becc6fbf3f0826625266dced32cc68df9 Mon Sep 17 00:00:00 2001
From: yoshiko2 <wenead99@gmail.com>
Date: Sat, 10 Jul 2021 15:01:56 +0800
Subject: [PATCH] Fix crash caused by custom site

---
 core.py | 50 +++++++++++++++++++++++++-------------------------
 1 file changed, 25 insertions(+), 25 deletions(-)

diff --git a/core.py b/core.py
index 1b517f2..8728a31 100755
--- a/core.py
+++ b/core.py
@@ -71,35 +71,35 @@ def get_data_from_json(file_number, filepath, conf: config.Config):  # 从JSON
 
     # default fetch order list, from the beginning to the end
     sources = conf.sources().split(',')
-
-    # if the input file name matches certain rules,
-    # move some web service to the beginning of the list
-    lo_file_number = file_number.lower()
-    if "carib" in sources and (re.match(r"^\d{6}-\d{3}", file_number)
-    ):
-        sources.insert(0, sources.pop(sources.index("carib")))
-    elif "avsox" in sources and (re.match(r"^\d{5,}", file_number) or
-        "heyzo" in lo_file_number
-    ):
-        sources.insert(0, sources.pop(sources.index("javdb")))
-        sources.insert(1, sources.pop(sources.index("avsox")))
-    elif "mgstage" in sources and (re.match(r"\d+\D+", file_number) or
-        "siro" in lo_file_number
-    ):
-        sources.insert(0, sources.pop(sources.index("mgstage")))
-    elif "fc2" in sources and ("fc2" in lo_file_number
-    ):
-        sources.insert(0, sources.pop(sources.index("javdb")))
-        sources.insert(1, sources.pop(sources.index("fc2")))
-    elif "dlsite" in sources and (
-        "rj" in lo_file_number or "vj" in lo_file_number
-    ):
-        sources.insert(0, sources.pop(sources.index("dlsite")))
+    if not len(conf.sources()) > 60:
+        # if the input file name matches certain rules,
+        # move some web service to the beginning of the list
+        lo_file_number = file_number.lower()
+        if "carib" in sources and (re.match(r"^\d{6}-\d{3}", file_number)
+        ):
+            sources.insert(0, sources.pop(sources.index("carib")))
+        elif "avsox" in sources and (re.match(r"^\d{5,}", file_number) or
+                                     "heyzo" in lo_file_number
+        ):
+            sources.insert(0, sources.pop(sources.index("javdb")))
+            sources.insert(1, sources.pop(sources.index("avsox")))
+        elif "mgstage" in sources and (re.match(r"\d+\D+", file_number) or
+                                       "siro" in lo_file_number
+        ):
+            sources.insert(0, sources.pop(sources.index("mgstage")))
+        elif "fc2" in sources and ("fc2" in lo_file_number
+        ):
+            sources.insert(0, sources.pop(sources.index("javdb")))
+            sources.insert(1, sources.pop(sources.index("fc2")))
+        elif "dlsite" in sources and (
+                "rj" in lo_file_number or "vj" in lo_file_number
+        ):
+            sources.insert(0, sources.pop(sources.index("dlsite")))
 
     json_data = {}
 
     if conf.multi_threading():
-        pool = ThreadPool(processes=11)
+        pool = ThreadPool(processes=len(conf.sources().split(',')))
 
         # Set the priority of multi-thread crawling and join the multi-thread queue
         for source in sources: