Merge branch 'master' of https://github.com/digitalmethodsinitiative/…

…4cat

common/lib/dataset.py

@@ -822,22 +822,25 @@ def update_version(self, version):
 
 		return updated > 0
 
-	def delete_parameter(self, parameter):
+	def delete_parameter(self, parameter, instant=True):
 		"""
 		Delete a parameter from the dataset metadata
 
 		:param string parameter:  Parameter to delete
+		:param bool instant:  Also delete parameters in this instance object?
 		:return bool:  Update successul?
 		"""
-		parameters = self.parameters
+		parameters = self.parameters.copy()
 		if parameter in parameters:
 			del parameters[parameter]
 		else:
 			return False
 
 		updated = self.db.update("datasets", where={"key": self.data["key"]},
 								 data={"parameters": json.dumps(parameters)})
-		self.parameters = parameters
+
+		if instant:
+			self.parameters = parameters
 
 		return updated > 0
 

datasources/reddit/search_reddit.py

@@ -41,7 +41,7 @@ class SearchReddit(SearchWithScope):
 					"careful** when using this privilege.",
 			"requires": "reddit.can_query_without_keyword"
 		},
-		"api_type": {
+		"pushshift_track": {
 			"type": UserInput.OPTION_CHOICE,
 			"help": "API version",
 			"options": {
@@ -149,7 +149,7 @@ def get_items_complex(self, query):
 		:return list:  Posts, sorted by thread and post ID, in ascending order
 		"""
 		scope = query.get("search_scope")
-		self.api_type = query.get("api_type", "regular")
+		self.api_type = query.get("pushshift_track", "regular")
 
 		# first, build the request parameters
 		if self.api_type == "regular":
@@ -646,7 +646,7 @@ def validate_query(query, request, user):
 				raise QueryParametersException("Please provide body queries that do not start with a minus sign.")
 
 		# URL queries are not possible (yet) for the beta API
-		if query.get("api_type") == "beta" and query.get("subject_url", None):
+		if query.get("pushshift_track") == "beta" and query.get("subject_url", None):
 			raise QueryParametersException("URL querying is not possible (yet) for the beta endpoint.")
 
 		# both dates need to be set, or none

datasources/telegram/search_telegram.py

@@ -127,12 +127,13 @@ def get_items(self, query):
                                        "creating it again from scratch.", is_final=True)
             return None
 
-        if not query.get("save-sensitive"):
-            self.dataset.delete_parameter("api_hash", instant=False)
-            self.dataset.delete_parameter("api_phone", instant=False)
-            self.dataset.delete_parameter("api_id", instant=False)
-
         results = asyncio.run(self.execute_queries())
+
+        if not query.get("save-sensitive"):
+            self.dataset.delete_parameter("api_hash", instant=True)
+            self.dataset.delete_parameter("api_phone", instant=True)
+            self.dataset.delete_parameter("api_id", instant=True)
+
         return results
 
     async def execute_queries(self):
@@ -506,6 +507,7 @@ def validate_query(query, request, user):
             "api_id": query.get("api_id"),
             "api_hash": query.get("api_hash"),
             "api_phone": query.get("api_phone"),
+            "save-sensitive": query.get("save-sensitive"),
             "min_date": min_date,
             "max_date": max_date
         }

processors/metrics/google_vision_api.py

@@ -46,7 +46,7 @@ def is_compatible_with(cls, module=None):
 
         :param module: Dataset or processor to determine compatibility with
         """
-        return module.type == "image-downloader"
+        return module.type.startswith("image-downloader")
 
     options = {
         "amount": {

processors/visualisation/download-telegram-images.py

@@ -28,7 +28,7 @@ class TelegramImageDownloader(BasicProcessor):
 
     Downloads attached images from Telegram messages and saves as zip archive
     """
-    type = "image-downloader"  # job type ID
+    type = "image-downloader-telegram"  # job type ID
     category = "Visual"  # category
     title = "Download images"  # title displayed in UI
     description = "Download images and compress as a zip file. May take a while to complete as images are downloaded " \

processors/visualisation/image_wall.py

@@ -80,7 +80,7 @@ def is_compatible_with(cls, module=None):
 
 		:param module: Dataset or processor to determine compatibility with
 		"""
-		return module.type == "image-downloader"
+		return module.type.startswith("image-downloader")
 
 	def process(self):
 		"""

processors/visualisation/pix-plot.py

@@ -98,7 +98,7 @@ def is_compatible_with(cls, module=None):
 
 		:param module: Dataset or processor to determine compatibility with
 		"""
-		return module.type == "image-downloader" and hasattr(config, 'PIXPLOT_SERVER') and config.PIXPLOT_SERVER
+		return module.type.startswith("image-downloader") and hasattr(config, 'PIXPLOT_SERVER') and config.PIXPLOT_SERVER
 
 	def process(self):
 		"""

processors/visualisation/rankflow.py

@@ -159,6 +159,10 @@ def process(self):
 			max_weight = max(max_weight, weight)
 			max_item_length = max(max_item_length, len(row["date"]))
 
+		if not items:
+			return self.dataset.finish_with_error("No items remain after filtering. Try disabling 'Remove items that "
+												  "do not occur...'.")
+
 		# determine per-period changes
 		# this is used for determining what colour to give to nodes, and
 		# visualise outlying items in the data

webtool/templates/result-details.html

@@ -67,7 +67,7 @@ <h2 class="blocktitle">
         <div class="fullwidth">
             <dt>API Credentials</dt>
             <dd>
-                {% for credential in has_credentials %}<span class="property-badge">{{ credential }}</span>{% endfor %}
+                {% for credential in has_credentials %}<span class="property-badge">{{ credential }}</span> {% endfor %}
                 (<a href="{{ url_for("erase_credentials_interactive", key=dataset.key) }}">erase</a>)
             </dd>
         </div>

webtool/views/api_tool.py

@@ -947,7 +947,7 @@ def datasource_call(datasource, action):
 		return error(400, error="Datasource '%s' has no call '%s'" % (datasource, action))
 
 	folder = backend.all_modules.datasources[datasource]["path"]
-	views_file = folder.joinpath("webtool", "views_misc.py")
+	views_file = folder.joinpath("webtool", "views.py")
 	if not views_file.exists():
 		return error(400, error="Datasource '%s' has no call '%s'" % (datasource, action))
 

webtool/views/views_dataset.py

@@ -9,7 +9,7 @@
 
 import flask
 import markdown
-from flask import render_template, abort, request, redirect, send_from_directory, flash, get_flashed_messages, \
+from flask import render_template, request, redirect, send_from_directory, flash, get_flashed_messages, \
     url_for, stream_with_context
 from flask_login import login_required, current_user
 
@@ -49,7 +49,7 @@ def show_page(page):
     page_path = page_folder + "/" + page + ".md"
 
     if not os.path.exists(page_path):
-        abort(404)
+        return error(404, error="Page not found")
 
     with open(page_path, encoding="utf-8") as file:
         page_raw = file.read()
@@ -117,7 +117,7 @@ def show_results(page):
     print(replacements)
 
     if not datasets and page != 1:
-        abort(404)
+        return error(404)
 
     pagination = Pagination(page, page_size, num_datasets)
     filtered = []
@@ -165,7 +165,7 @@ def get_mapped_result(key):
     try:
         dataset = DataSet(key=key, db=db)
     except TypeError:
-        abort(404)
+        return error(404, error="Dataset not found.")
 
     if dataset.is_private and not (current_user.is_admin or dataset.owner == current_user.get_id()):
         return error(403, error="This dataset is private.")
@@ -176,7 +176,7 @@ def get_mapped_result(key):
 
     if not hasattr(dataset.get_own_processor(), "map_item"):
         # cannot map without a mapping method
-        abort(404)
+        return error(404, error="File not found.")
 
     mapper = dataset.get_own_processor().map_item
 
@@ -216,7 +216,7 @@ def view_log(key):
     try:
         dataset = DataSet(key=key, db=db)
     except TypeError:
-        return error(404, "Dataset not found.")
+        return error(404, error="Dataset not found.")
 
     if dataset.is_private and not (current_user.is_admin or dataset.owner == current_user.get_id()):
         return error(403, error="This dataset is private.")
@@ -553,10 +553,10 @@ def delete_dataset_interactive(key):
 @login_required
 def erase_credentials_interactive(key):
     """
-	Erase sensitive parameters from dataset
+    Erase sensitive parameters from dataset
 
-	Removes all parameters starting with `api_`. This heuristic could be made
-	more expansive if more fine-grained control is required.
+    Removes all parameters starting with `api_`. This heuristic could be made
+    more expansive if more fine-grained control is required.
 
     Uses code from corresponding API endpoint, but redirects to a normal page
     rather than returning JSON as the API does, so this can be used for