From 50153f342e8ebdcc388f371b0087025c0f88cd87 Mon Sep 17 00:00:00 2001 From: Rosanna Milner Date: Mon, 5 Aug 2024 14:48:53 +0100 Subject: [PATCH] Adding credibility signals feature Add assistant text classifier UI with html formatting first commit added new sagas starting to add accordion display Displaying gcloud cred signals tidied up list of genre, topic, persuasion added typography machince generated text service added fixed subjectivity displaying subjective sentences displaying correct subjective sentences tidied display of credibillity signals using stored secrets for kinit services removed classifiers from URL test box added code for checking logged in for pfc preventing pfc from being called if not logged in removing unnnecessary imports adding subj tooltip and pfc limit refactored credibility signals added messages for after logging in login message and passing text to semantic search as variable captialised genres added assistant.json text moved tooltips out to cardheader pfc and mgt sagas work after login typos added comments added embedding message moved credsignals grid to correct card removed console.log message removed unnecessary tooltip log moved text to assistant.json tidied code counting categories in wrapfunc counting categories in wrapfunc change counting categories in wrapfunc change Subjectivty confidence colour blue refactoring pertech code refactor for correct persuasion techniques added/removed comments updated tooltip text remove console.log fixes for persuasion techniques fixes for subjectivity edited keyword for semantic search changed mgt.pred to use keyword --- .../components/NavItems/tools/Assistant.json | 273 +++--- .../NavItems/Assistant/Assistant.jsx | 26 +- .../AssistantApiHandlers/useAssistantApi.jsx | 150 +++ .../AssistantCheckStatus.jsx | 18 + .../ExtractedSourceCredibilityDBKFDialog.jsx | 46 +- .../AssistantCredibilitySignals.jsx | 895 ++++++++++++++++++ .../AssistantTextClassification.jsx | 340 +++++++ .../AssistantTextResult.jsx | 10 +- .../AssistantTextSpanClassification.jsx | 399 ++++++++ .../assistantTextResultStyle.css | 7 + .../AssistantScrapeResults/assistantUtils.jsx | 226 ++++- .../components/ResultDisplayItem.jsx | 13 +- .../NavItems/tools/SemanticSearch/index.jsx | 17 +- src/redux/actions/tools/assistantActions.jsx | 104 ++ src/redux/reducers/assistantReducer.jsx | 36 + src/redux/sagas/assistantSaga.jsx | 162 +++- 16 files changed, 2547 insertions(+), 175 deletions(-) create mode 100644 src/components/NavItems/Assistant/AssistantScrapeResults/AssistantCredibilitySignals.jsx create mode 100644 src/components/NavItems/Assistant/AssistantScrapeResults/AssistantTextClassification.jsx create mode 100644 src/components/NavItems/Assistant/AssistantScrapeResults/AssistantTextSpanClassification.jsx create mode 100644 src/components/NavItems/Assistant/AssistantScrapeResults/assistantTextResultStyle.css diff --git a/public/locales/en/components/NavItems/tools/Assistant.json b/public/locales/en/components/NavItems/tools/Assistant.json index 8971587d8..ca61e0ae2 100644 --- a/public/locales/en/components/NavItems/tools/Assistant.json +++ b/public/locales/en/components/NavItems/tools/Assistant.json @@ -1,116 +1,161 @@ { - "assistant_title": "Assistant", - "assistant_intro": "The assistant will help you to analyse a webpage, an image or a video file and suggest which WeVerify tools are useful for each case", - "assistant_choose": "Choose what you want to analyse", - "assistant_webpage_header": "Webpage link", - "assistant_webpage_text": "Insert the link of a webpage, and the assistant will suggest the most useful tools for the contents of the given page.", - "assistant_file_header": "Local file", - "assistant_file_text": "The assistant will suggest the most useful tools depending on if the file is a video or image", - "assistant_choose_tool": "Choose the tool you want to use", - "upload_video": "Video", - "upload_image": "Image", - "assistant_urlbox": "URL", - "assistant_give_url": "Give the URL of the page to analyse", - "assistant_paste_url": "Paste the URL here", - "please_give_a_correct_link": "The link provided is incorrect or not supported.", - "button_analyse": "Analyse", - "button_clean": "Remove", - "save": "Save", - "source_credibility_title": "Source Credibility", - "source_credibility_byline": "The source has been found as part of a credibility check", - "link_explorer_title": "Link Explorer", - "link_explorer_byline": "The following URLs have been extracted from the page, and their domains have been checked for credibility", - "text_title": "Text", - "text_intro": "The following text has been found on the page", - "dbkf_title": "DBKF Check", - "expand_text": "Expand Text", - "dbkf_error": "An issue has occurred when trying to connect to the database of known fakes. Some results may be omitted from this page. If the problem persists, please contact support.", - "sc_failed": "The source credibility check has failed. Some results may not be displayed. If the problem persists, please contact support.", - "link_tooltip": "What is this?
The scores displayed in this section give an indication of how reliable the web source listed can be considered. Scores range from 0-100.

In most cases, the source checked for credibility will be the URL domain. In some cases where this is not useful a more relevant part of the URL has been checked against. In all cases, the full URL and the domain against which the source credibility check was carried on are listed in the form URL : domain-for-check.

How are these calculated?
Various institutions analyse sources (particularly domains) across the web to evaluate how reliable they are likely to be based on various metrics.
Here, datasets assessing web domain reliability have been pulled mainly from Open Sources and the WeVerify DBKF database.
Since the metrics or scores from each of these sources do not necessarily overlap and cannot easily be compared, all scores from any given institution are mapped to a single number between 0-100.
For any URL, a list of all institutions which gave the relevant domain a score has been listed, along with the (mapped) results they gave.
The final result displayed for any domain is the lowest score from the various sources.
", - "media_title": "Media", - "images_label": "Images", - "videos_label": "Videos", - "media_found": "The following media has been found on the page", - "media_below": "Select the media you would like to verify", - "media_to_process": "Media to Process", - "assistant_error": "An unexpected assistant error has occurred. If the problem persists, contact support.", - "things_you_can_do_header": "Potential Tools", - "things_you_can_do": "Below are the tools you can use on this media type", - "navbar_analysis_image": "Image Analysis", - "navbar_analysis_video": "Video Analysis", - "navbar_keyframes": "Keyframes", - "navbar_thumbnails": "Thumbnails", - "navbar_twitter": "Tw. search", - "navbar_magnifier": "Magnifier", - "navbar_metadata": "Metadata", - "navbar_rights": "Video rights", - "navbar_forensic": "Forensic", - "navbar_ocr": "OCR", - "navbar_twitter_sna": "Fact Check", - "navbar_assistant": "Assistant", - "assistant_help_title": "Assistant Help", - "assistant_help_1": "
The WeVerify toolkit (this plugin) has multiple tools which can help in the verification of content on social networks, designed to help journalists save time and be more efficient in their fact checking tasks.
A full list of these tools can be found in the tutorial section of this plugin.
Given the multiple tools and services available, the weverify assistant has been designed to guide users to the services available to them given the nature of the content they would like to check.

", - "assistant_help_2": "How does the assistant work?
Users can choose to upload their own media to be checked (an image or a video), in which case verification tools on the plugin which support media upload will be listed.
An alternative option is to enter a URL which needs verification or fact checking.
Once a URL in entered, the assistant attempts to extract any text, images or videos it can find on the page.
Any extracted text or media are then used to suggest the potential actions which can be carried out. Currently, there are three major components to this:More information on each of these can be found in the links below.

", - "assistant_help_3": "Which URLS are supported?
Currently, there is dedicated support for links for the following types of URLs: Other generic links can also be entered and the assistant will attempt to retrieve what it can. However, any results (particularly on the extraction of images and text) on generic links may be much more generalised.

", - "assistant_help_4": "More information
For a detailed breakdown of the type of URLs supported by the assistant, see this page
For a detailed breakdown of the tools and checks run by the assistant, see: this page", - "enter_url": "Enter URL", - "mode_label": "URL mode", - "url_text": "URL Text", - "url_media": "URL Media", - "media_text": "Extracted Media Text", - "download_video": "The video has been extracted but cannot be processed using the direct URL. Please open it in a new tab using the link below, download it, and use the assistant to see which tools can be used on the downloaded video.", - "text_warning": "Warnings found against text. See warning box.", - "image_warning": "Warnings found against image. See warning box.", - "warning_title": "Warning", - "warning_subtitle": "Some elements of this content have been flagged", - "status_title": "Status", - "status_subtitle": "Note: one or more of the automated checks have resulted in an error. Some results may be omitted from this page.", - "named_entity_title": "Text entities", - "dbkf_image_warning": "The image from this URL has matched against the following from the database of known fakes with a similarity score of", - "dbkf_video_warning": "The video from this URL has matched against the following from the database of known fakes with a similarity score of", - "dbkf_text_warning": "The text from this URL has matched against the following from the database of known fakes", - "domain_scope": "Domain:", - "account_scope": "Account:", - "labelled_as": "has been labelled as:", - "commented_as": "with the following comment:", - "hp_warning": "The text from this URL has returned an unusually high hyperpartisan score of", - "image_analysis_text": "Retrieve contextual information about the image", - "video_analysis_text": "Retrieve contextual information about the video", - "keyframes_text": "Fragment the video", - "thumbnails_text": "Reverse search on thumbnails from the video", - "magnifier_text": "Examine the image thoroughly", - "metadata_text": "Retrieve image/video metadata", - "rights_text": "Retrieve information solely about video rights", - "forensic_text": "Detect any image manipulation", - "ocr_text": "Extract any text from the image", - "hyperpartisan_title": "Hyperpartisan service", - "dbkf_text_title": "Database of known fakes text search", - "dbkf_media_title": "Database of known fakes media search", - "ne_title": "Named entity search", - "ocr_title": "OCR search", - "source_cred_title": "Source credibility service", - "mt_title": "Machine translation service", - "translate": "Translate (beta)", - "copy_link": "Copy link", - "archive_link": "Archive", - "this": "This ", - "source_credibility_warning_domain": "The domain has been mentioned in a lookup against:", - "source_credibility_warning_account": "account has been mentioned in a lookup against:", - "text_tooltip": "What is this?
Any text which can be extracted from the given URL is displayed here

How is this used?
Various text processing tools are run against this text: For more information, see: this page", - "media_tooltip": "What is this?
Any images or videos which can be extracted from the URL have been displayed here.

What can I do with these?
For any selected image/video, a list of relevant tools from the plugin which can process this media are given. Selecting any of these will redirect the media to the selected tool.
Alongside this, the assistant will reverse search the media against the database of known fakes. If any results are found, these will be displayed in the Warnings section.

For more information, see: this page", - "sc_tooltip": "What is this?
For every URL entered, we run either the domain (if this is a non social media URL) or the account (if this is a social media URL) against our URL domain analysis service. The results of this check are shown here.

What can I do with these?
The domain analysis service intends to collect information about a domain/account from multiple sources. From here, we intend to inform the user whether any of the sources we collect from hold information on the domain/account of the URL entered. The result is split into three types: Since this is information collected from multiple sources, we have given the source we have taken the details from and if provided, any evidence the source itself has given so users can check the validity of claims made about a domain. For more information on sources and data collection, see this page", - "source_cred_popup_header_domain": "The domain has been found in the following reports listed by:", - "source_cred_popup_header_account": "The account has been found in the following reports listed by:", - "url_domain_analysis": "URL Domain Analysis", - "fact_checker": "Fact checker", - "warning": "Warning", - "mentions": "Mentions", - "assistant_error_instagram": "The assistant has failed to retrieve this instagram post. Please open this URL in a new tab (you may need to log in to instagram), then use the ASSISTANT FOR CURRENT PAGE button to retry.", - "assistant_error_server_error": "The assistant could not process the link provided due to a problem with the server, please try again later. If the problem persists please contact support.", - "assistant_error_connection_error": "The assistant is having issues connecting to the server, please try again later.", - "extracted_urls": "Extracted URLs", - "extracted_urls_url_domain_analysis": "Extracted URLs with URL Domain Analysis", - "extracted_urls_url_domain_analysis_failed": "Extracted URLs with URL Domain Analysis has failed.", - "extracted_urls_tooltip": "What is this?
For every URL extracted from the original URL, we run either the domain (if this is a non social media URL) or the account (if this is a social media URL) against our URL domain analysis service. The results of this check are shown here.

What can I do with these?
The domain analysis service intends to collect information about a domain/account from multiple sources. From here, we intend to inform the user whether any of the sources we collect from hold information on the domain/account of the URL entered. The result is split into three types: Since this is information collected from multiple sources, we have given the source we have taken the details from and if provided, any evidence the source itself has given so users can check the validity of claims made about a domain. For more information on sources and data collection, see this page", - "embedding_not_supported":"The Assistant could not display this video content." + "assistant_title": "Assistant", + "assistant_intro": "The assistant will help you to analyse a webpage, an image or a video file and suggest which WeVerify tools are useful for each case", + "assistant_choose": "Choose what you want to analyse", + "assistant_webpage_header": "Webpage link", + "assistant_webpage_text": "Insert the link of a webpage, and the assistant will suggest the most useful tools for the contents of the given page.", + "assistant_file_header": "Local file", + "assistant_file_text": "The assistant will suggest the most useful tools depending on if the file is a video or image", + "assistant_choose_tool": "Choose the tool you want to use", + "upload_video": "Video", + "upload_image": "Image", + "assistant_urlbox": "URL", + "assistant_give_url": "Give the URL of the page to analyse", + "assistant_paste_url": "Paste the URL here", + "please_give_a_correct_link": "The link provided is incorrect or not supported.", + "button_analyse": "Analyse", + "button_clean": "Remove", + "save": "Save", + "source_credibility_title": "Source Credibility", + "source_credibility_byline": "The source has been found as part of a credibility check", + "link_explorer_title": "Link Explorer", + "link_explorer_byline": "The following URLs have been extracted from the page, and their domains have been checked for credibility", + "text_title": "Text", + "text_intro": "The following text has been found on the page", + "dbkf_title": "DBKF Check", + "expand_text": "Expand Text", + "dbkf_error": "An issue has occurred when trying to connect to the database of known fakes. Some results may be omitted from this page. If the problem persists, please contact support.", + "sc_failed": "The source credibility check has failed. Some results may not be displayed. If the problem persists, please contact support.", + "link_tooltip": "What is this?
The scores displayed in this section give an indication of how reliable the web source listed can be considered. Scores range from 0-100.

In most cases, the source checked for credibility will be the URL domain. In some cases where this is not useful a more relevant part of the URL has been checked against. In all cases, the full URL and the domain against which the source credibility check was carried on are listed in the form URL : domain-for-check.

How are these calculated?
Various institutions analyse sources (particularly domains) across the web to evaluate how reliable they are likely to be based on various metrics.
Here, datasets assessing web domain reliability have been pulled mainly from Open Sources and the WeVerify DBKF database.
Since the metrics or scores from each of these sources do not necessarily overlap and cannot easily be compared, all scores from any given institution are mapped to a single number between 0-100.
For any URL, a list of all institutions which gave the relevant domain a score has been listed, along with the (mapped) results they gave.
The final result displayed for any domain is the lowest score from the various sources.
", + "media_title": "Media", + "images_label": "Images", + "videos_label": "Videos", + "media_found": "The following media has been found on the page", + "media_below": "Select the media you would like to verify", + "media_to_process": "Media to Process", + "assistant_error": "An unexpected assistant error has occurred. If the problem persists, contact support.", + "things_you_can_do_header": "Potential Tools", + "things_you_can_do": "Below are the tools you can use on this media type", + "navbar_analysis_image": "Image Analysis", + "navbar_analysis_video": "Video Analysis", + "navbar_keyframes": "Keyframes", + "navbar_thumbnails": "Thumbnails", + "navbar_twitter": "Tw. search", + "navbar_magnifier": "Magnifier", + "navbar_metadata": "Metadata", + "navbar_rights": "Video rights", + "navbar_forensic": "Forensic", + "navbar_ocr": "OCR", + "navbar_twitter_sna": "Fact Check", + "navbar_assistant": "Assistant", + "assistant_help_title": "Assistant Help", + "assistant_help_1": "
The WeVerify toolkit (this plugin) has multiple tools which can help in the verification of content on social networks, designed to help journalists save time and be more efficient in their fact checking tasks.
A full list of these tools can be found in the tutorial section of this plugin.
Given the multiple tools and services available, the weverify assistant has been designed to guide users to the services available to them given the nature of the content they would like to check.

", + "assistant_help_2": "How does the assistant work?
Users can choose to upload their own media to be checked (an image or a video), in which case verification tools on the plugin which support media upload will be listed.
An alternative option is to enter a URL which needs verification or fact checking.
Once a URL in entered, the assistant attempts to extract any text, images or videos it can find on the page.
Any extracted text or media are then used to suggest the potential actions which can be carried out. Currently, there are three major components to this:More information on each of these can be found in the links below.

", + "assistant_help_3": "Which URLS are supported?
Currently, there is dedicated support for links for the following types of URLs: Other generic links can also be entered and the assistant will attempt to retrieve what it can. However, any results (particularly on the extraction of images and text) on generic links may be much more generalised.

", + "assistant_help_4": "More information
For a detailed breakdown of the type of URLs supported by the assistant, see this page
For a detailed breakdown of the tools and checks run by the assistant, see: this page", + "enter_url": "Enter URL", + "mode_label": "URL mode", + "url_text": "URL Text", + "url_media": "URL Media", + "media_text": "Extracted Media Text", + "download_video": "The video has been extracted but cannot be processed using the direct URL. Please open it in a new tab using the link below, download it, and use the assistant to see which tools can be used on the downloaded video.", + "text_warning": "Warnings found against text. See warning box.", + "image_warning": "Warnings found against image. See warning box.", + "warning_title": "Warning", + "warning_subtitle": "Some elements of this content have been flagged", + "status_title": "Status", + "status_subtitle": "Note: one or more of the automated checks have resulted in an error. Some results may be omitted from this page.", + "named_entity_title": "Text entities", + "dbkf_image_warning": "The image from this URL has matched against the following from the database of known fakes with a similarity score of", + "dbkf_video_warning": "The video from this URL has matched against the following from the database of known fakes with a similarity score of", + "dbkf_text_warning": "The text from this URL has matched against the following from the database of known fakes", + "domain_scope": "Domain:", + "account_scope": "Account:", + "labelled_as": "has been labelled as:", + "commented_as": "with the following comment:", + "hp_warning": "The text from this URL has returned an unusually high hyperpartisan score of", + "image_analysis_text": "Retrieve contextual information about the image", + "video_analysis_text": "Retrieve contextual information about the video", + "keyframes_text": "Fragment the video", + "thumbnails_text": "Reverse search on thumbnails from the video", + "magnifier_text": "Examine the image thoroughly", + "metadata_text": "Retrieve image/video metadata", + "rights_text": "Retrieve information solely about video rights", + "forensic_text": "Detect any image manipulation", + "ocr_text": "Extract any text from the image", + "hyperpartisan_title": "Hyperpartisan service", + "dbkf_text_title": "Database of known fakes text search", + "dbkf_media_title": "Database of known fakes media search", + "ne_title": "Named entity search", + "ocr_title": "OCR search", + "source_cred_title": "Source credibility service", + "mt_title": "Machine translation service", + "translate": "Translate (beta)", + "copy_link": "Copy link", + "archive_link": "Archive", + "this": "This ", + "source_credibility_warning_domain": "The domain has been mentioned in a lookup against:", + "source_credibility_warning_account": "account has been mentioned in a lookup against:", + "text_tooltip": "What is this?
Any text which can be extracted from the given URL is displayed here

How is this used?
Various text processing tools are run against this text: For more information, see: this page", + "media_tooltip": "What is this?
Any images or videos which can be extracted from the URL have been displayed here.

What can I do with these?
For any selected image/video, a list of relevant tools from the plugin which can process this media are given. Selecting any of these will redirect the media to the selected tool.
Alongside this, the assistant will reverse search the media against the database of known fakes. If any results are found, these will be displayed in the Warnings section.

For more information, see: this page", + "sc_tooltip": "What is this?
For every URL entered, we run either the domain (if this is a non social media URL) or the account (if this is a social media URL) against our URL domain analysis service. The results of this check are shown here.

What can I do with these?
The domain analysis service intends to collect information about a domain/account from multiple sources. From here, we intend to inform the user whether any of the sources we collect from hold information on the domain/account of the URL entered. The result is split into three types: Since this is information collected from multiple sources, we have given the source we have taken the details from and if provided, any evidence the source itself has given so users can check the validity of claims made about a domain. For more information on sources and data collection, see this page", + "source_cred_popup_header_domain": "The domain has been found in the following reports listed by:", + "source_cred_popup_header_account": "The account has been found in the following reports listed by:", + "url_domain_analysis": "URL Domain Analysis", + "fact_checker": "Fact checker", + "warning": "Warning", + "mentions": "Mentions", + "assistant_error_instagram": "The assistant has failed to retrieve this instagram post. Please open this URL in a new tab (you may need to log in to instagram), then use the ASSISTANT FOR CURRENT PAGE button to retry.", + "assistant_error_server_error": "The assistant could not process the link provided due to a problem with the server, please try again later. If the problem persists please contact support.", + "assistant_error_connection_error": "The assistant is having issues connecting to the server, please try again later.", + "extracted_urls": "Extracted URLs", + "extracted_urls_url_domain_analysis": "Extracted URLs with URL Domain Analysis", + "extracted_urls_url_domain_analysis_failed": "Extracted URLs with URL Domain Analysis has failed.", + "extracted_urls_tooltip": "What is this?
For every URL extracted from the original URL, we run either the domain (if this is a non social media URL) or the account (if this is a social media URL) against our URL domain analysis service. The results of this check are shown here.

What can I do with these?
The domain analysis service intends to collect information about a domain/account from multiple sources. From here, we intend to inform the user whether any of the sources we collect from hold information on the domain/account of the URL entered. The result is split into three types: Since this is information collected from multiple sources, we have given the source we have taken the details from and if provided, any evidence the source itself has given so users can check the validity of claims made about a domain. For more information on sources and data collection, see this page", + "embedding_not_supported":"The Assistant could not display this video content.", + "assistant_video_download_action": "Download video", + "assistant_video_download_action_description": "Click here to download the video, to your machine. The download video can then be used in the assistant to access a wider range of video analysis services.", + "credibility_signals": "Credibility Signals", + "credibility_signals_tooltip": "What is this?
This section displays various credibility signals derived by applying AI classifiers to the extracted text of a given page.

What can I do with these?
The results of these credibility signals give a detailed overview of the extracted text which allows the reader to decide how credible or reliable the source is. For more information on these signals, please see this page.", + "importance_tooltip": "The background of highlighted sentences varies depending on the detection algorithm's rating of its importance.", + "confidence_tooltip_technique": "The background of the detected techniques varies depending on the detection algorithm's confidence.", + "confidence_tooltip_sentence": "The background of highlighted sentences varies depending on the detection algorithm's confidence.", + "highlight_important_sentence": "Highlight important sentences", + "low_importance": "Low importance", + "high_importance": "High importance", + "low_confidence": "Low confidence", + "high_confidence": "High confidence", + "colour_scale": "The colour scale is shown below:", + "news_framing": "Topic", + "news_framing_tooltip": "Identify the topics used in the extracted text. A topic is the perspective under which an issue or a piece of news is presented. A total of 9 different topics are considered.", + "news_genre": "Genre", + "news_genre_tooltip": "Determine whether the text is most likely to be an opinion piece, objective news reporting, or satire.", + "persuasion_techniques": "Persuasion Techniques", + "persuasion_techniques_tooltip": "Identify the persuasion techniques of the extracted text. This service annotates particular sentences within the text that use these techniques. A total of 23 different techniques are considered.", + "detected_techniques": "Detected techniques", + "no_detected_categories": "No detected categories", + "subjectivity": "Subjectivity", + "subjectivity_tooltip": "Identify the subjective sentences of the extracted text.", + "subjective_sentences_detected": "Subjective sentences detected", + "no_subjective_sentences_detected": "No subjective sentences detected", + "previous_fact_checks": "Previous Fact-Checks", + "previous_fact_checks_tooltip": "The Fact Check Semantic Search tool identifies whether the extracted text has previously been detected in a fact check database. It displays the most recent matches, up to a maximum of 5. This tool is currently only accessible to beta users.", + "more_details": "For more details see", + "semantic_search_title": "Fact Check Semantic Search", + "failed_to_load": "Failed to load", + "previous_fact_checks_found": "Top 5 previous fact-checks found", + "login_required": "Please log in as a beta user to see results", + "reanalyse_url": "Please reanalyse URL to see results", + "semantic_search_result_claim": "Claim:", + "semantic_search_result_title": "Title:", + "semantic_search_result_translated_from": "Translated from", + "semantic_search_result_see_original": "See original", + "semantic_search_result_english_translation": "Show English Translation", + "semantic_search_rating": "Rating:", + "machine_generated_text": "Machine Generated Text", + "machine_generated_text_tooltip": "Determine whether the extracted text has been written by human or machine. This tool is currently only accessible to beta users.", + "highly_likely_human": "Highly likely human written with score ", + "likely_human": "Likely human written with score ", + "likely_machine": "Likely machine generated with score ", + "highly_likely_machine": "Highly likely machine generated with score " } \ No newline at end of file diff --git a/src/components/NavItems/Assistant/Assistant.jsx b/src/components/NavItems/Assistant/Assistant.jsx index a3ab327cc..cdcc397d9 100644 --- a/src/components/NavItems/Assistant/Assistant.jsx +++ b/src/components/NavItems/Assistant/Assistant.jsx @@ -19,6 +19,7 @@ import AssistantSCResults from "./AssistantScrapeResults/AssistantSCResults"; import AssistantTextResult from "./AssistantScrapeResults/AssistantTextResult"; import AssistantUrlSelected from "./AssistantUrlSelected"; import AssistantWarnings from "./AssistantScrapeResults/AssistantWarnings"; +import AssistantCredSignals from "./AssistantScrapeResults/AssistantCredibilitySignals"; import { cleanAssistantState, @@ -52,6 +53,13 @@ const Assistant = () => { //third party check states const neResult = useSelector((state) => state.assistant.neResultCategory); + const newsFramingResult = useSelector( + (state) => state.assistant.newsFramingResult, + ); + const newsGenreResult = useSelector( + (state) => state.assistant.newsGenreResult, + ); + const hpResult = useSelector((state) => state.assistant.hpResult); // source credibility const positiveSourceCred = useSelector( @@ -79,6 +87,13 @@ const Assistant = () => { (state) => state.assistant.dbkfMediaMatchFail, ); const neFailState = useSelector((state) => state.assistant.neFail); + const newsFramingFailState = useSelector( + (state) => state.assistant.newsFramingFail, + ); + const newsGenreFailState = useSelector( + (state) => state.assistant.newsGenreFail, + ); + // const mtFailState = useSelector(state => state.assistant.mtFail) //local state const [formInput, setFormInput] = useState(inputUrl); @@ -163,7 +178,9 @@ const Assistant = () => { {scFailState || dbkfTextFailState || dbkfMediaFailState || - neFailState ? ( + neFailState || + newsFramingFailState || + newsGenreFailState ? ( @@ -233,6 +250,12 @@ const Assistant = () => { ) : null} + + {text ? ( + + + + ) : null} @@ -240,5 +263,4 @@ const Assistant = () => { ); }; - export default Assistant; diff --git a/src/components/NavItems/Assistant/AssistantApiHandlers/useAssistantApi.jsx b/src/components/NavItems/Assistant/AssistantApiHandlers/useAssistantApi.jsx index 472d54fa1..3a243c51c 100644 --- a/src/components/NavItems/Assistant/AssistantApiHandlers/useAssistantApi.jsx +++ b/src/components/NavItems/Assistant/AssistantApiHandlers/useAssistantApi.jsx @@ -76,16 +76,166 @@ export default function assistantApiCalls() { return result.data; }; + const MAX_NUM_RETRIES = 3; + + /** + * Calls an async function that throws an exception when it fails, will retry for numMaxRetries + * @param numMaxRetries Number of times the function will be retried + * @param asyncFunc The async function to call + * @param errorFunc Called when asyncFunc throws an error when there are additional retries + * @returns {Promise<*>} Output of asyncFunc + */ + async function callAsyncWithNumRetries( + numMaxRetries, + asyncFunc, + errorFunc = null, + ) { + for (let retryCount = 0; retryCount < numMaxRetries; retryCount++) { + try { + return await asyncFunc(); + } catch (e) { + if (retryCount + 1 >= MAX_NUM_RETRIES) { + throw e; + } else { + if (errorFunc) errorFunc(retryCount, e); + } + } + } + } + + const callNewsFramingService = async (text) => { + return await callAsyncWithNumRetries( + MAX_NUM_RETRIES, + async () => { + const result = await axios.post( + assistantEndpoint + "gcloud/news-framing-clfr", + { text: text }, + ); + return result.data; + }, + (numTries) => { + console.log( + "Could not connect to news framing service, tries " + + (numTries + 1) + + "/" + + MAX_NUM_RETRIES, + ); + }, + ); + }; + + const callNewsGenreService = async (text) => { + return await callAsyncWithNumRetries( + MAX_NUM_RETRIES, + async () => { + const result = await axios.post( + assistantEndpoint + "gcloud/news-genre-clfr", + { text: text }, + ); + return result.data; + }, + (numTries) => { + console.log( + "Could not connect to news genre service, tries " + + (numTries + 1) + + "/" + + MAX_NUM_RETRIES, + ); + }, + ); + }; + + const callPersuasionService = async (text) => { + return await callAsyncWithNumRetries( + MAX_NUM_RETRIES, + async () => { + const result = await axios.post( + assistantEndpoint + "gcloud/persuasion-span-clfr", + { text: text }, + ); + return result.data; + }, + (numTries) => { + console.log( + "Could not connect to persuasion service, tries " + + (numTries + 1) + + "/" + + MAX_NUM_RETRIES, + ); + }, + ); + }; + const callOcrScriptService = async () => { const result = await axios.get(assistantEndpoint + "gcloud/ocr-scripts"); return result.data; }; + const callSubjectivityService = async (text) => { + const result = await axios.post(assistantEndpoint + "dw/subjectivity", { + content: text, + }); + + return result.data; + }; + + const callPrevFactChecksService = async (text) => { + return await callAsyncWithNumRetries( + MAX_NUM_RETRIES, + async () => { + const result = await axios.get( + assistantEndpoint + + "kinit/prev-fact-checks" + + "?text=" + + encodeURIComponent(text), // max URL length is 2048 characters + ); + return result.data; + }, + (numTries) => { + console.log( + "Could not connect to previous fact checks service, tries " + + (numTries + 1) + + "/" + + MAX_NUM_RETRIES, + ); + }, + ); + }; + + const callMachineGeneratedTextService = async (text) => { + return await callAsyncWithNumRetries( + MAX_NUM_RETRIES, + async () => { + const result = await axios.get( + assistantEndpoint + + "kinit/machine-generated-text" + + "?text=" + + encodeURIComponent(text), // max URL length is 2048 characters + ); + return result.data; + }, + (numTries) => { + console.log( + "Could not connect to machine generated text service, tries " + + (numTries + 1) + + "/" + + MAX_NUM_RETRIES, + ); + }, + ); + }; + return { callAssistantScraper, callSourceCredibilityService, callNamedEntityService, callOcrService, callOcrScriptService, + callNewsFramingService, + callNewsGenreService, + callPersuasionService, + callSubjectivityService, + callPrevFactChecksService, + callMachineGeneratedTextService, }; } diff --git a/src/components/NavItems/Assistant/AssistantCheckResults/AssistantCheckStatus.jsx b/src/components/NavItems/Assistant/AssistantCheckResults/AssistantCheckStatus.jsx index 522fe8b3a..b1978e721 100644 --- a/src/components/NavItems/Assistant/AssistantCheckResults/AssistantCheckStatus.jsx +++ b/src/components/NavItems/Assistant/AssistantCheckResults/AssistantCheckStatus.jsx @@ -34,11 +34,29 @@ const AssistantCheckStatus = () => { const neTitle = keyword("ne_title"); const neFailState = useSelector((state) => state.assistant.neFail); + const newsFramingTitle = "news topic"; + const newsFramingFailState = useSelector( + (state) => state.assistant.newsFramingFail + ); + + const newsGenreTitle = "news genre"; + const newsGenreFailState = useSelector( + (state) => state.assistant.newsGenreFail + ); + + const persuasionTitle = "persuasion"; + const persuasionFailState = useSelector( + (state) => state.assistant.persuasionFail + ); + const failStates = [ { title: scTitle, failed: scFailState }, { title: dbkfMediaTitle, failed: dbkfMediaFailState }, { title: dbkfTextTitle, failed: dbkfTextFailState }, { title: neTitle, failed: neFailState }, + { title: newsFramingTitle, failed: newsFramingFailState }, + { title: newsGenreTitle, failed: newsGenreFailState }, + { title: persuasionTitle, failed: persuasionFailState }, ]; return ( diff --git a/src/components/NavItems/Assistant/AssistantCheckResults/ExtractedSourceCredibilityDBKFDialog.jsx b/src/components/NavItems/Assistant/AssistantCheckResults/ExtractedSourceCredibilityDBKFDialog.jsx index 4992f0403..45ab449e7 100644 --- a/src/components/NavItems/Assistant/AssistantCheckResults/ExtractedSourceCredibilityDBKFDialog.jsx +++ b/src/components/NavItems/Assistant/AssistantCheckResults/ExtractedSourceCredibilityDBKFDialog.jsx @@ -139,28 +139,30 @@ const ExtractedSourceCredibilityDBKFDialog = ({ ? sourceCredibilityResults.map((value, key) => ( }> - {sourceCredibilityResults[ - key - ].credibilityScope.includes("/") ? ( - - {` ${keyword("this")}`} - {getUrlTypeFromCredScope( - value.credibilityScope, - )} - {` ${keyword( - "source_credibility_warning_account", - )} ${" "}${value.credibilitySource}`} - - ) : sourceCredibilityResults[key] - .credibilityScope ? ( - - {` ${keyword( - "source_cred_popup_header_domain", - )} ${ - sourceCredibilityResults[key] - .credibilitySource - } `} - + {sourceCredibilityResults[key] ? ( + sourceCredibilityResults[ + key + ].credibilityScope.includes("/") ? ( + + {` ${keyword("this")}`} + {getUrlTypeFromCredScope( + value.credibilityScope, + )} + {` ${keyword( + "source_credibility_warning_account", + )} ${" "}${value.credibilitySource}`} + + ) : sourceCredibilityResults[key] + .credibilityScope ? ( + + {` ${keyword( + "source_cred_popup_header_domain", + )} ${ + sourceCredibilityResults[key] + .credibilitySource + } `} + + ) : null ) : null} diff --git a/src/components/NavItems/Assistant/AssistantScrapeResults/AssistantCredibilitySignals.jsx b/src/components/NavItems/Assistant/AssistantScrapeResults/AssistantCredibilitySignals.jsx new file mode 100644 index 000000000..9d102bee4 --- /dev/null +++ b/src/components/NavItems/Assistant/AssistantScrapeResults/AssistantCredibilitySignals.jsx @@ -0,0 +1,895 @@ +import React, { useState } from "react"; +import { useSelector } from "react-redux"; + +import Box from "@mui/material/Box"; +import Card from "@mui/material/Card"; +import { CardHeader, CircularProgress, Link, styled } from "@mui/material"; +import CardContent from "@mui/material/CardContent"; +import Grid from "@mui/material/Grid"; +import Accordion from "@mui/material/Accordion"; +import AccordionSummary from "@mui/material/AccordionSummary"; +import AccordionDetails from "@mui/material/AccordionDetails"; +import ExpandMoreIcon from "@mui/icons-material/ExpandMore"; +import Typography from "@mui/material/Typography"; +import { i18nLoadNamespace } from "components/Shared/Languages/i18nLoadNamespace"; +import useMyStyles from "../../../Shared/MaterialUiStyles/useMyStyles"; +import Tooltip from "@mui/material/Tooltip"; +import HelpOutlineOutlinedIcon from "@mui/icons-material/HelpOutlineOutlined"; +import AssistantTextClassification from "./AssistantTextClassification"; +import AssistantTextSpanClassification from "./AssistantTextSpanClassification"; +import ResultDisplayItem from "components/NavItems/tools/SemanticSearch/components/ResultDisplayItem"; +import dayjs from "dayjs"; +import LocaleData from "dayjs/plugin/localeData"; +import localizedFormat from "dayjs/plugin/localizedFormat"; + +import Collapse from "@mui/material/Collapse"; +import Divider from "@mui/material/Divider"; +import { + ExpandLessOutlined, + ExpandMoreOutlined, + Remove, +} from "@mui/icons-material"; +import TranslateIcon from "@mui/icons-material/Translate"; +import IconButton from "@mui/material/IconButton"; +import FileCopyOutlined from "@mui/icons-material/FileCopy"; + +import { useNavigate } from "react-router-dom"; +import { getLanguageName } from "../../../Shared/Utils/languageUtils"; + +const renderEntityKeys = (entities) => { + // tidy array into readable string + let entitiesString = Object.keys(entities) + .toString() + .replace("Important_Sentence", "") + .replaceAll(",", ", ") + .replaceAll("_", " ") + .trim(); + + // remove beginning and last hanging commas + if (entitiesString.slice(0, 2) === ", ") { + entitiesString = entitiesString.substring(2, entitiesString.length); + } + if (entitiesString.slice(-1) === ",") { + entitiesString = entitiesString.substring(0, entitiesString.length - 1); + } + + return entitiesString; +}; + +const round = (number, decimalPlaces) => { + return (Math.round(number * 100) / 100).toFixed(decimalPlaces); +}; + +const calculateSubjectivity = (sentences) => { + let scoresSUBJ = []; + for (let i = 0; i < sentences.length; i++) { + if (sentences[i].label == "SUBJ") { + scoresSUBJ.push(Number(sentences[i].score)); + } + } + + return [ + scoresSUBJ.length, + [" (", scoresSUBJ.length, "/", sentences.length, ")"] + .toString() + .replaceAll(",", ""), + ]; +}; + +const getExpandIcon = (loading, fail, done = null, role = null) => { + if (loading || fail || done || (role && !role.includes("BETA_TESTER"))) { + // "done" is for when subjectivityDone = true and subjectivityResult.entities.length + return ; + } else { + return ; + } +}; + +const renderCollapse = ( + classes, + setDisplayOrigLang, + displayOrigLang, + textLang, + sharedKeyword, + keyword, + text, + displayExpander, + expanded, + setExpanded, +) => { + return ( + + + + + setDisplayOrigLang(!displayOrigLang)} + > + {textLang} + + + { + navigator.clipboard.writeText(text); + }} + > + + + + {textLang && textLang !== "en" && textLang !== "" ? ( + + + window.open( + "https://translate.google.com/?sl=auto&text=" + + encodeURIComponent(text) + + "&op=translate", + "_blank", + ) + } + > + + + + ) : null} + + + {displayExpander ? ( + expanded ? ( + { + setExpanded(!expanded); + }} + /> + ) : ( + { + setExpanded(!expanded); + }} + /> + ) + ) : null} + + + + ); +}; + +const renderCollapsePrevFactChecks = ( + classes, + displayExpander, + expanded, + setExpanded, + navigate, + keyword, +) => { + const handleClick = (path) => { + // instead need to set parameter then load text in SemanticSearch/index.jsx + navigate("/app/" + path + "/assistantText"); + }; + + return ( + + + + + <> + + + +

+ {keyword("more_details")}{" "} + handleClick("tools/semanticSearch")} + > + {keyword("semantic_search_title")} + + + + + {displayExpander ? ( + expanded ? ( + { + setExpanded(!expanded); + }} + /> + ) : ( + { + setExpanded(!expanded); + }} + /> + ) + ) : null} + + + + ); +}; + +const AssistantCredSignals = () => { + const keyword = i18nLoadNamespace("components/NavItems/tools/Assistant"); + const sharedKeyword = i18nLoadNamespace("components/Shared/utils"); + const classes = useMyStyles(); + + // displaying expanded text in AccordionDetails + const [displayOrigLang, setDisplayOrigLang] = useState(true); + const [displayExpander, setDisplayExpander] = useState(true); + const [expanded, setExpanded] = useState(true); + + // one accordion open at once + const [expandedAccordion, setExpandedAccordion] = React.useState("false"); + const handleChange = (panel) => (event, newExpanded) => { + setExpandedAccordion(newExpanded ? panel : false); + }; + + //style disabled accordion + const StyledAccordion = styled(Accordion)(({ theme }) => ({ + ".Mui-disabled": { + opacity: "1 !important", + background: "white", + }, + })); + + // assistant media states + const text = useSelector((state) => state.assistant.urlText); + const textLang = useSelector((state) => state.assistant.textLang); + const textHtmlMap = useSelector((state) => state.assistant.urlTextHtmlMap); + + // news framing (topic) + const newsFramingTitle = keyword("news_framing"); + const newsFramingResult = useSelector( + (state) => state.assistant.newsFramingResult, + ); + const newsFramingLoading = useSelector( + (state) => state.assistant.newsFramingLoading, + ); + const newsFramingDone = useSelector( + (state) => state.assistant.newsFramingDone, + ); + const newsFramingFail = useSelector( + (state) => state.assistant.newsFramingFail, + ); + + // news genre + const newsGenreTitle = keyword("news_genre"); + const newsGenreResult = useSelector( + (state) => state.assistant.newsGenreResult, + ); + const newsGenreLoading = useSelector( + (state) => state.assistant.newsGenreLoading, + ); + const newsGenreDone = useSelector((state) => state.assistant.newsGenreDone); + const newsGenreFail = useSelector((state) => state.assistant.newsGenreFail); + + // persuasion techniques + const persuasionTitle = keyword("persuasion_techniques"); + const persuasionResult = useSelector( + (state) => state.assistant.persuasionResult, + ); + const persuasionLoading = useSelector( + (state) => state.assistant.persuasionLoading, + ); + const persuasionDone = useSelector((state) => state.assistant.persuasionDone); + const persuasionFail = useSelector((state) => state.assistant.persuasionFail); + + // subjectivity + const subjectivityTitle = keyword("subjectivity"); + const subjectivityResult = useSelector( + (state) => state.assistant.subjectivityResult, + ); + const subjectivityLoading = useSelector( + (state) => state.assistant.subjectivityLoading, + ); + const subjectivityDone = useSelector( + (state) => state.assistant.subjectivityDone, + ); + const subjectivityFail = useSelector( + (state) => state.assistant.subjectivityFail, + ); + + // previous fact checks + const prevFactChecksTitle = keyword("previous_fact_checks"); + const prevFactChecksResult = useSelector( + (state) => state.assistant.prevFactChecksResult, + ); + const prevFactChecksLoading = useSelector( + (state) => state.assistant.prevFactChecksLoading, + ); + const prevFactChecksDone = useSelector( + (state) => state.assistant.prevFactChecksDone, + ); + const prevFactChecksFail = useSelector( + (state) => state.assistant.prevFactChecksFail, + ); + // checking if user logged in + const role = useSelector((state) => state.userSession.user.roles); + // date information + dayjs.extend(LocaleData); + dayjs.extend(localizedFormat); + const globalLocaleData = dayjs.localeData(); + // for navigating to Semantic Search with text + const navigate = useNavigate(); + + // machine generated text + const machineGeneratedTextTitle = keyword("machine_generated_text"); + const machineGeneratedTextResult = useSelector( + (state) => state.assistant.machineGeneratedTextResult, + ); + const machineGeneratedTextLoading = useSelector( + (state) => state.assistant.machineGeneratedTextLoading, + ); + const machineGeneratedTextDone = useSelector( + (state) => state.assistant.machineGeneratedTextDone, + ); + const machineGeneratedTextFail = useSelector( + (state) => state.assistant.machineGeneratedTextFail, + ); + + return ( + + + + {keyword("credibility_signals")} + + } + action={ + // tooltip +
" + + keyword("news_framing") + + "
" + + keyword("news_framing_tooltip") + + "

" + + keyword("news_genre") + + "
" + + keyword("news_genre_tooltip") + + "

" + + keyword("persuasion_techniques") + + "
" + + keyword("persuasion_techniques_tooltip") + + "

" + + keyword("subjectivity") + + "
" + + keyword("subjectivity_tooltip") + + "

" + + keyword("previous_fact_checks") + + "
" + + keyword("previous_fact_checks_tooltip") + + "

" + + keyword("machine_generated_text") + + "
" + + keyword("machine_generated_text_tooltip"), + }} + /> + } + classes={{ tooltip: classes.assistantTooltip }} + > + + + } + /> + + + {/* News Framing/Topic */} + + + + + + {newsFramingTitle} + + + + {newsFramingLoading && ( + + )} + {newsFramingFail && ( + + {keyword("failed_to_load")} + + )} + {newsFramingDone && ( + + {renderEntityKeys(newsFramingResult.entities)} + + )} + + + + + + {newsFramingDone && ( +
+ + + + {renderCollapse( + classes, + setDisplayOrigLang, + displayOrigLang, + textLang, + sharedKeyword, + keyword, + text, + displayExpander, + expanded, + setExpanded, + )} +
+ )} + + + + {/* News Genre */} + + + + + + {newsGenreTitle} + + + + {newsGenreLoading && } + {newsGenreFail && ( + + {keyword("failed_to_load")} + + )} + {newsGenreDone && ( + + {renderEntityKeys(newsGenreResult.entities)} + + )} + + + + + + {newsGenreDone && ( +
+ + + + {renderCollapse( + classes, + setDisplayOrigLang, + displayOrigLang, + textLang, + sharedKeyword, + keyword, + text, + displayExpander, + expanded, + setExpanded, + )} +
+ )} + + + + {/* Persuasion Techniques */} + + + + + + {persuasionTitle} + + + + {persuasionLoading && ( + + )} + {persuasionFail && ( + + {keyword("failed_to_load")} + + )} + {persuasionDone && ( + + {renderEntityKeys(persuasionResult.entities)} + + )} + + + + + + {persuasionDone && ( +
+ + + + {renderCollapse( + classes, + setDisplayOrigLang, + displayOrigLang, + textLang, + sharedKeyword, + keyword, + text, + displayExpander, + expanded, + setExpanded, + )} +
+ )} + + + + {/* Subjectivity */} + + + + + + {subjectivityTitle} + + + + {subjectivityLoading && ( + + )} + {subjectivityFail && ( + + {keyword("failed_to_load")} + + )} + {subjectivityDone && ( + + {calculateSubjectivity(subjectivityResult.sentences)[0] != + 0 + ? keyword("subjective_sentences_detected") + + calculateSubjectivity(subjectivityResult.sentences)[1] + : keyword("no_subjective_sentences_detected")} + + )} + + + + + + {subjectivityDone && ( +
+ + + + {renderCollapse( + classes, + setDisplayOrigLang, + displayOrigLang, + textLang, + sharedKeyword, + keyword, + text, + displayExpander, + expanded, + setExpanded, + )} +
+ )} + + + + {/* Previous fact-checks */} + + + + + + {prevFactChecksTitle} + + + + + {role.includes("BETA_TESTER") && prevFactChecksLoading && ( + + )} + {role.includes("BETA_TESTER") && prevFactChecksFail && ( + + {keyword("failed_to_load")} + + )} + {role.includes("BETA_TESTER") && + prevFactChecksDone && + prevFactChecksResult && ( + + {keyword("previous_fact_checks_found")} + + )} + {role.includes("BETA_TESTER") && + !prevFactChecksDone && + !prevFactChecksLoading && + !prevFactChecksFail && + !prevFactChecksResult && ( + + {keyword("reanalyse_url")} + {/* should now be obselete as saga is re run */} + + )} + {!role.includes("BETA_TESTER") && ( + + {keyword("login_required")} + + )} + + + + + + {prevFactChecksDone && role.includes("BETA_TESTER") && ( +
+ + {prevFactChecksResult + ? prevFactChecksResult.map((resultItem) => { + // date in correct format + const date = resultItem.published_at.slice(0, 10); + + return ( + + ); + }) + : null} + + {renderCollapsePrevFactChecks( + classes, + displayExpander, + expanded, + setExpanded, + navigate, + keyword, + )} +
+ )} + + + + {/* Machine Generated Text */} + + + + + + {machineGeneratedTextTitle} + + + + + {role.includes("BETA_TESTER") && + machineGeneratedTextLoading && ( + + )} + {role.includes("BETA_TESTER") && machineGeneratedTextFail && ( + + {keyword("failed_to_load")} + + )} + {role.includes("BETA_TESTER") && + machineGeneratedTextDone && + machineGeneratedTextResult && ( + + {keyword(machineGeneratedTextResult.pred)} + {round(machineGeneratedTextResult.score, 4)} + + )} + {role.includes("BETA_TESTER") && + !machineGeneratedTextDone && + !machineGeneratedTextLoading && + !machineGeneratedTextFail && + !machineGeneratedTextResult && ( + + {keyword("reanalyse_url")} + {/* should now be obselete as saga is re run */} + + )} + {!role.includes("BETA_TESTER") && ( + + {keyword("login_required")} + + )} + + + + + + + + ); +}; +export default AssistantCredSignals; diff --git a/src/components/NavItems/Assistant/AssistantScrapeResults/AssistantTextClassification.jsx b/src/components/NavItems/Assistant/AssistantScrapeResults/AssistantTextClassification.jsx new file mode 100644 index 000000000..5e1c92bca --- /dev/null +++ b/src/components/NavItems/Assistant/AssistantScrapeResults/AssistantTextClassification.jsx @@ -0,0 +1,340 @@ +import React, { useState } from "react"; +import { useDispatch } from "react-redux"; + +import Card from "@mui/material/Card"; +import { + CardHeader, + Checkbox, + FormControlLabel, + List, + ListItem, + ListItemText, +} from "@mui/material"; +import CardContent from "@mui/material/CardContent"; +import Divider from "@mui/material/Divider"; +import Grid from "@mui/material/Grid"; +import HelpOutlineOutlinedIcon from "@mui/icons-material/HelpOutlineOutlined"; +import Tooltip from "@mui/material/Tooltip"; +import Typography from "@mui/material/Typography"; +import useMyStyles from "../../../Shared/MaterialUiStyles/useMyStyles"; +import { i18nLoadNamespace } from "components/Shared/Languages/i18nLoadNamespace"; +import { + interpRgb, + rgbToString, + rgbToLuminance, + rgbListToGradient, + treeMapToElements, + wrapPlainTextSpan, +} from "./assistantUtils"; + +import "./assistantTextResultStyle.css"; + +export default function AssistantTextClassification({ + text, + classification, + titleText = "Detected Class", + importantSentenceKey = "Important_Sentence", + helpDescription = "", + configs = { + confidenceThresholdLow: 0.8, + confidenceThresholdHigh: 1.0, + importanceThresholdLow: 0.0, + importanceThresholdHigh: 1.0, + confidenceRgbLow: [175, 9, 193], + confidenceRgbHigh: [34, 0, 255], + importanceRgbLow: [221, 222, 7], + importanceRgbHigh: [228, 25, 25], + }, + textHtmlMap = null, + subjectivity = false, +}) { + const classes = useMyStyles(); + const dispatch = useDispatch(); + const keyword = i18nLoadNamespace("components/NavItems/tools/Assistant"); + + // subjectivity or not + let toolipText; + let textLow, textHigh; + let rgbLow, rgbHigh; + if (subjectivity) { + toolipText =

{keyword("confidence_tooltip_sentence")}

; + textLow = keyword("low_confidence"); + textHigh = keyword("high_confidence"); + rgbLow = configs.confidenceRgbLow; + rgbHigh = configs.confidenceRgbHigh; + } else { + toolipText =

{keyword("importance_tooltip")}

; + textLow = keyword("low_importance"); + textHigh = keyword("high_importance"); + rgbLow = configs.importanceRgbLow; + rgbHigh = configs.importanceRgbHigh; + } + + const importanceTooltipContent = ( + + ); + const confidenceTooltipContent = ( + + ); + + let filteredCategories = {}; + let filteredSentences = []; + + const [doHighlightSentence, setDoHighlightSentence] = useState(true); + const handleHighlightSentences = (event) => { + setDoHighlightSentence(event.target.checked); + }; + + // Separate important sentences from categories, filter by threshold + for (let label in classification) { + if (label === importantSentenceKey) { + // Filter sentences above importanceThresholdLow + const sentenceIndices = classification[label]; + for (let i = 0; i < sentenceIndices.length; i++) { + if (sentenceIndices[i].score >= configs.importanceThresholdLow) { + filteredSentences.push(sentenceIndices[i]); + } + } + } else { + //Filter categories above confidenceThreshold + if (classification[label][0].score >= configs.confidenceThresholdLow) { + filteredCategories[label] = classification[label]; + } + } + } + + // disabled category box for Subjectivity classifier + // subjectivty or not + let width = 12; + if (!subjectivity) { + width = 9; + } + + return ( + + + + + {!subjectivity ? ( + + + + + + + + } + /> + + + {filteredSentences.length > 0 ? ( + + } + label={keyword("highlight_important_sentence")} + /> + ) : null} + + + + ) : null} + + ); +} + +export function CategoriesList({ + categories, + noCategoriesText, + thresholdLow, + thresholdHigh, + rgbLow, + rgbHigh, +}) { + if (categories.length < 1) return

{noCategoriesText}

; + + let output = []; + let index = 0; + for (const category in categories) { + if (index > 0) { + output.push(); + } + let backgroundRgb = interpRgb( + categories[category][0].score, + thresholdLow, + thresholdHigh, + rgbLow, + rgbHigh, + ); + let bgLuminance = rgbToLuminance(backgroundRgb); + let textColour = "white"; + if (bgLuminance > 0.7) textColour = "black"; + + output.push( + + + , + ); + index++; + } + return {output}; +} + +/* +Takes input from topic classifier and convert them into html sentence highlighting + */ +export function ClassifiedText({ + text, + spanIndices, + highlightSpan, + tooltipText, + thresholdLow, + thresholdHigh, + rgbLow, + rgbHigh, + textHtmlMap = null, +}) { + let output = text; //Defaults to text output + + function wrapHighlightedText(spanText, spanInfo) { + const spanScore = spanInfo.score; + let backgroundRgb = interpRgb( + spanScore, + thresholdLow, + thresholdHigh, + rgbLow, + rgbHigh, + ); + let bgLuminance = rgbToLuminance(backgroundRgb); + let textColour = "white"; + if (bgLuminance > 0.7) textColour = "black"; + + return ( + + + {spanText} + + + ); + } + + if (highlightSpan && spanIndices.length > 0) { + if (textHtmlMap) { + // Text formatted & highlighted + output = treeMapToElements( + text, + textHtmlMap, + spanIndices, + wrapHighlightedText, + ); + } else { + // Plaintex & highlighted + output = wrapPlainTextSpan(text, spanIndices, wrapHighlightedText); + } + } else if (textHtmlMap) { + // Text formatted but not highlighted + output = treeMapToElements(text, textHtmlMap); + } + + return {output}; +} + +export function ColourGradientScale({ textLow, textHigh, rgbList }) { + return ( + <> + + + + {textLow} + + + + + {textHigh} + + + +
+ + ); +} + +export function ColourGradientTooltipContent({ + description = "", + colourScaleText, + textLow = "Low", + textHigh = "High", + rgbLow = [0, 0, 0], + rgbHigh = [255, 255, 255], +}) { + return ( +
+ {description} +

{colourScaleText}

+ +
+ ); +} diff --git a/src/components/NavItems/Assistant/AssistantScrapeResults/AssistantTextResult.jsx b/src/components/NavItems/Assistant/AssistantScrapeResults/AssistantTextResult.jsx index b7605b61a..fd210be56 100644 --- a/src/components/NavItems/Assistant/AssistantScrapeResults/AssistantTextResult.jsx +++ b/src/components/NavItems/Assistant/AssistantScrapeResults/AssistantTextResult.jsx @@ -7,12 +7,12 @@ import { CardHeader } from "@mui/material"; import CardContent from "@mui/material/CardContent"; import Collapse from "@mui/material/Collapse"; import Divider from "@mui/material/Divider"; + import { ExpandLessOutlined, ExpandMoreOutlined, WarningOutlined, } from "@mui/icons-material"; -import FormatQuoteIcon from "@mui/icons-material/FormatQuote"; import Grid from "@mui/material/Grid"; import HelpOutlineOutlinedIcon from "@mui/icons-material/HelpOutlineOutlined"; import LinearProgress from "@mui/material/LinearProgress"; @@ -53,16 +53,20 @@ const AssistantTextResult = () => { const textBox = document.getElementById("element-to-check"); const [expanded, setExpanded] = useState(false); const [displayOrigLang, setDisplayOrigLang] = useState(true); - const [displayExpander, setDisplayExpander] = useState(false); + const [displayExpander, setDisplayExpander] = useState(true); - // figure out if component displaying text needs collapse icon useEffect(() => { + // if (translatedText) { + // setDisplayOrigLang(false); + // } const elementToCheck = document.getElementById("element-to-check"); if (elementToCheck.offsetHeight < elementToCheck.scrollHeight) { setDisplayExpander(true); } if (textHtmlMap !== null) { + // HTML text is contained in an xml document, we need to parse it and + // extract all contents in the
node# setTextHtmlOutput(treeMapToElements(text, textHtmlMap)); } }, [textBox]); diff --git a/src/components/NavItems/Assistant/AssistantScrapeResults/AssistantTextSpanClassification.jsx b/src/components/NavItems/Assistant/AssistantScrapeResults/AssistantTextSpanClassification.jsx new file mode 100644 index 000000000..a3ab96c3e --- /dev/null +++ b/src/components/NavItems/Assistant/AssistantScrapeResults/AssistantTextSpanClassification.jsx @@ -0,0 +1,399 @@ +import React, { useState } from "react"; +import { useDispatch } from "react-redux"; + +import { Button } from "@mui/material"; + +import Card from "@mui/material/Card"; +import { CardHeader, Chip, List, ListItem, ListItemText } from "@mui/material"; +import CardContent from "@mui/material/CardContent"; +import Divider from "@mui/material/Divider"; + +import Grid from "@mui/material/Grid"; +import HelpOutlineOutlinedIcon from "@mui/icons-material/HelpOutlineOutlined"; +import Tooltip from "@mui/material/Tooltip"; +import Typography from "@mui/material/Typography"; +import useMyStyles from "../../../Shared/MaterialUiStyles/useMyStyles"; +import { i18nLoadNamespace } from "components/Shared/Languages/i18nLoadNamespace"; + +import { + interpRgb, + rgbToString, + rgbToLuminance, + treeMapToElements, + mergeSpanIndices, + wrapPlainTextSpan, +} from "./assistantUtils"; +import { ColourGradientTooltipContent } from "./AssistantTextClassification"; +import { styled } from "@mui/system"; + +// Had to create a custom styled span as the default style attribute does not support +// :hover metaclass +const StyledSpan = styled("span")(); + +export default function AssistantTextSpanClassification({ + text, + classification, + titleText = "Detected Class", + importantSentenceKey = "Important_Sentence", + helpDescription = "", + configs = { + confidenceThresholdLow: 0.8, + confidenceThresholdHigh: 1.0, + importanceThresholdLow: 0.0, + importanceThresholdHigh: 1.0, + confidenceRgbLow: [175, 9, 193], + confidenceRgbHigh: [34, 0, 255], + importanceRgbLow: [221, 222, 7], + importanceRgbHigh: [228, 25, 25], + }, + textHtmlMap = null, +}) { + const classes = useMyStyles(); + const dispatch = useDispatch(); + const keyword = i18nLoadNamespace("components/NavItems/tools/Assistant"); + const tooltipTextLowThreshold = keyword("low_confidence"); + const tooltipTextHighThreshold = keyword("high_confidence"); + + const confidenceTooltipContent = ( + + ); + + const [doHighlightSentence, setDoHighlightSentence] = useState(true); + const handleHighlightSentences = (event) => { + setDoHighlightSentence(event.target.checked); + }; + + function filterLabelsWithMinThreshold(classification, minThreshold) { + let filteredLabels = {}; + for (let label in classification) { + let spanList = []; + for (let i = 0; i < classification[label].length; i++) { + if (classification[label][i].score >= minThreshold) { + spanList.push(classification[label][i]); + } + } + + if (spanList.length > 0) { + filteredLabels[label] = spanList; + } + } + return filteredLabels; + } + + let filteredClassification = filterLabelsWithMinThreshold( + classification, + configs.confidenceThresholdLow, + ); + + const [currentLabel, setCurrentLabel] = useState(null); + + function handleCategorySelect(categoryKey) { + setCurrentLabel(categoryKey); + } + + // finding categories and their spans with scores, and the text for each category + let categories = {}; + let categoriesText = {}; + + // combine all sentences for an overall category + let collectFilteredClassification = {}; + for (let category in filteredClassification) { + collectFilteredClassification[category] = { + [category]: filteredClassification[category], + }; + } + const allCategoriesLabel = "all"; + collectFilteredClassification[allCategoriesLabel] = filteredClassification; + + // wrap function for calculating spanhighlights and categories + function wrapHighlightedText(spanText, spanInfo, spanStart, spandEnd) { + let backgroundRgb = [210, 210, 210]; + let backgroundRgbHover = [255, 100, 100]; + let textColour = "black"; + + let techniqueContent = []; + techniqueContent.push(

{keyword("detected_techniques")}

); + + for (let persuasionTechnique in spanInfo.techniques) { + const techniqueScore = spanInfo.techniques[persuasionTechnique]; + + // collect category information for highlighted spans + // let span = { + // indices: [spanStart, spandEnd], + // score: techniqueScore, + // }; + if (categories[persuasionTechnique]) { + categories[persuasionTechnique].push({ + indices: [spanStart, spandEnd], + score: techniqueScore, + }); + } else { + categories[persuasionTechnique] = [ + { + indices: [spanStart, spandEnd], + score: techniqueScore, + }, + ]; + } + + let techniqueBackgroundRgb = interpRgb( + techniqueScore, + configs.confidenceThresholdLow, + configs.confidenceThresholdHigh, + configs.confidenceRgbLow, + configs.confidenceRgbHigh, + ); + let bgLuminance = rgbToLuminance(techniqueBackgroundRgb); + let techniqueTextColour = "white"; + if (bgLuminance > 0.7) techniqueTextColour = "black"; + techniqueContent.push( +
+ {persuasionTechnique.replaceAll("_", " ")} +
, + ); + } + techniqueContent.push(keyword("confidence_tooltip_technique")); + + let techniquesTooltip = ( + + ); + + // Append highlighted text + return ( + + + {spanText} + + + ); + } + + // find the highlighted spans for each category and overall category + for (let collection in collectFilteredClassification) { + let output; + + // Classification variable is a map of categories where each one has a list of classified spans, we + // have to invert that so that we have a list of spans that contains all categories in that span + let mergedSpanIndices = mergeSpanIndices( + collectFilteredClassification[collection], + ); + + if (doHighlightSentence && mergedSpanIndices.length > 0) { + if (textHtmlMap) { + // Text formatted & highlighted + output = treeMapToElements( + text, + textHtmlMap, + mergedSpanIndices, + wrapHighlightedText, + ); + } else { + // Plaintex & highlighted + output = wrapPlainTextSpan( + text, + mergedSpanIndices, + wrapHighlightedText, + ); + } + } else if (textHtmlMap) { + // Text formatted but not highlighted + output = treeMapToElements(text, textHtmlMap); + } + + categoriesText[collection] = output; + } + + // console.log("categories=", categories); + // console.log("categoriesText=", categoriesText); + + // remove duplicate spans from array of categories + // duplicates occur as categories are counted then repeated for category allCategoriesLabel + let uniqueCategories = {}; + for (let cat in categories) { + uniqueCategories[cat] = categories[cat].filter((value, index) => { + const _value = JSON.stringify(value); + return ( + index === + categories[cat].findIndex((obj) => { + return JSON.stringify(obj) === _value; + }) + ); + }); + } + + // console.log("uniqueCategories=", uniqueCategories); + + return ( + + + + + + + + + + +
+ } + /> + + + + + + + ); +} + +export function CategoriesListToggle({ + categories, + thresholdLow, + thresholdHigh, + rgbLow, + rgbHigh, + noCategoriesText, + allCategoriesLabel, + onCategoryChange = () => {}, +}) { + if (categories.length < 1) return

{noCategoriesText}

; + + let output = []; + let index = 0; + const [currentCategory, setCurrentCategory] = useState(null); + + function handleCategorySelect(categoryLabel) { + if (categoryLabel === currentCategory) { + setCurrentCategory(null); + onCategoryChange(null); + } else { + setCurrentCategory(categoryLabel); + onCategoryChange(categoryLabel); + } + } + + function handleCategoryHover(categoryLabel) { + onCategoryChange(categoryLabel); + } + + function handleCategoryOut() { + onCategoryChange(currentCategory); + } + + for (const category in categories) { + // don't display overall category + if (category == allCategoriesLabel) { + continue; + } + + if (index > 0) { + output.push(); + } + + let backgroundRgb = interpRgb( + categories[category][0].score, + thresholdLow, + thresholdHigh, + rgbLow, + rgbHigh, + ); + let bgLuminance = rgbToLuminance(backgroundRgb); + let textColour = "white"; + if (bgLuminance > 0.7) textColour = "black"; + + const itemText = category.replaceAll("_", " "); + const itemChip = ( + + ); + + output.push( + handleCategoryHover(category)} + onMouseLeave={() => handleCategoryOut()} + onClick={() => handleCategorySelect(category)} + > + + , + ); + index++; + } + + return {output}; +} + +export function MultiCategoryClassifiedText({ + categoriesText, + currentLabel, + allCategoriesLabel, +}) { + // Filter for selecting all labels (currentLabel == null) or just a single label + const category = + currentLabel !== null && currentLabel in categoriesText + ? currentLabel + : allCategoriesLabel; + + let output = categoriesText[category]; + + return {output}; +} diff --git a/src/components/NavItems/Assistant/AssistantScrapeResults/assistantTextResultStyle.css b/src/components/NavItems/Assistant/AssistantScrapeResults/assistantTextResultStyle.css new file mode 100644 index 000000000..05068361f --- /dev/null +++ b/src/components/NavItems/Assistant/AssistantScrapeResults/assistantTextResultStyle.css @@ -0,0 +1,7 @@ +.testBgSpan { + background: green; +} + +.testBgSpan:hover { + background: red; +} diff --git a/src/components/NavItems/Assistant/AssistantScrapeResults/assistantUtils.jsx b/src/components/NavItems/Assistant/AssistantScrapeResults/assistantUtils.jsx index ac656344a..46b23403d 100644 --- a/src/components/NavItems/Assistant/AssistantScrapeResults/assistantUtils.jsx +++ b/src/components/NavItems/Assistant/AssistantScrapeResults/assistantUtils.jsx @@ -1,4 +1,79 @@ import React from "react"; +import _ from "lodash"; +import { ColourGradientTooltipContent } from "./AssistantTextClassification"; +import Tooltip from "@mui/material/Tooltip"; + +/** + * Interpolate RGB between an arbitrary range + * @param value + * @param low + * @param high + * @param rgbLow RGB Value represented as an array e.g. [255, 255, 255] for white + * @param rgbHigh RGB Value represented as an array e.g. [255, 255, 255] for white + * @returns {*[]} + */ +export const interpRgb = (value, low, high, rgbLow, rgbHigh) => { + let interp = value; + if (value < low) interp = low; + if (value > high) interp = high; + interp = (interp - low) / (high - low); + + let output = []; + for (let i = 0; i < rgbLow.length; i++) { + let channelLow = rgbLow[i]; + let channelHigh = rgbHigh[i]; + let delta = channelHigh - channelLow; + output.push(channelLow + delta * interp); + } + + return output; +}; + +/** + * Converts an array-based RGB representation to rgba() format used by CSS + * @param rgb RGB value represented as an array e.g. [255, 255, 255] for white + * @returns {string} + */ +export const rgbToString = (rgb) => { + return "rgba(" + rgb[0] + "," + rgb[1] + "," + rgb[2] + ",1)"; +}; + +/** + * Calculates the luminance of an RGB colour + * @param rgb RGB value represented as an array e.g. [255, 255, 255] for white + * @returns {number} + */ +export const rgbToLuminance = (rgb) => { + let rgbNorm = rgb.map((c) => c / 255); + let rgbLinear = rgbNorm.map((c) => { + if (c <= 0.04045) { + return c / 12.92; + } else { + return Math.pow((c + 0.055) / 1.055, 2.4); + } + }); + let luminance = + rgbLinear[0] * 0.2126 + rgbLinear[1] * 0.7152 + rgbLinear[2] * 0.0722; + return luminance; +}; + +/** + * Generate CSS gradient from a list of RGB values + * @param rgbList List RGB value represented as an array e.g. [[0,0,0], [255, 255, 255]] for a black to white gradient + * @returns {string} + */ +export const rgbListToGradient = (rgbList) => { + let gradientStr = ""; + for (let i = 0; i < rgbList.length; i++) { + let colourStr = rgbToString(rgbList[i]); + let percentageStr = Math.round((i / (rgbList.length - 1)) * 100).toString(); + + if (i > 0) gradientStr += ","; + gradientStr += colourStr + " " + percentageStr + "%"; + } + const output = "linear-gradient(90deg, " + gradientStr + ")"; + return output; +}; function treeMapToElementsRecursive( text, @@ -10,34 +85,34 @@ function treeMapToElementsRecursive( if ("span" in treeElem) { const span = treeElem.span; if (spanHighlightIndices === null) { - console.log("No span highlight: ", text.substring(span.start, span.end)); + // console.log("No span highlight: ", text.substring(span.start, span.end)); childElems.push(text.substring(span.start, span.end)); } else { - console.log("Span highlight: ", text.substring(span.start, span.end)); + // console.log("Span highlight: ", text.substring(span.start, span.end)); let currentIndex = span.start; for (let i = 0; i < spanHighlightIndices.length; i++) { const hSpan = spanHighlightIndices[i]; - console.log( - "Matching span", - span.start, - span.end, - hSpan.indices[0], - hSpan.indices[1], - ); + // console.log( + // "Matching span", + // span.start, + // span.end, + // hSpan.indices[0], + // hSpan.indices[1], + // ); const hSpanStart = hSpan.indices[0]; const hSpanEnd = hSpan.indices[1]; if ( (span.start <= hSpanStart && hSpanStart <= span.end) || (span.start <= hSpanEnd && hSpanEnd <= span.end) ) { - //If there's an overlap - console.log( - "Found lapping span ", - span.start, - span.end, - hSpanStart, - hSpanEnd, - ); + // //If there's an overlap + // console.log( + // "Found lapping span ", + // span.start, + // span.end, + // hSpanStart, + // hSpanEnd, + // ); // If span doesn't start before the current index if (hSpanStart > currentIndex) { @@ -48,15 +123,20 @@ function treeMapToElementsRecursive( hSpanStart < span.start ? span.start : hSpanStart; const boundedEnd = hSpanEnd > span.end ? span.end : hSpanEnd; if (wrapFunc) { - console.log("Wrapping: ", text.substring(boundedStart, boundedEnd)); + // console.log("Wrapping: ", text.substring(boundedStart, boundedEnd)); childElems.push( - wrapFunc(text.substring(boundedStart, boundedEnd), hSpan), + wrapFunc( + text.substring(boundedStart, boundedEnd), + hSpan, + boundedStart, + boundedEnd, + ), ); } else { - console.log( - "Not wrapping: ", - text.substring(boundedStart, boundedEnd), - ); + // console.log( + // "Not wrapping: ", + // text.substring(boundedStart, boundedEnd), + // ); childElems.push(text.substring(boundedStart, boundedEnd)); } @@ -81,17 +161,17 @@ function treeMapToElementsRecursive( ), ); } + return React.createElement(treeElem.tag, null, childElems); } /** - * Combines text and html tree to generate dynamic DOM elements. A handler function can be provided in order to - * insert dynamic elements around each text span. + * * @param text * @param mapping - * @param spanHighlightIndices Indices of text spans that should be wrapped, in the format of [{indices: [start, end]}, ...] - * @param wrapFunc Called for each span indices in spanHighlightIndices the function should take the form of: (text, spanInfo): Element => {} - * @returns {React.DetailedReactHTMLElement, HTMLInputElement>|*} + * @param spanHighlightIndices + * @param wrapFunc function(spanText, spanInfo) + * @returns Array of plaintext or components directly renderable by react */ export const treeMapToElements = ( text, @@ -100,13 +180,101 @@ export const treeMapToElements = ( wrapFunc = null, ) => { if (mapping) { - return treeMapToElementsRecursive( + let output = treeMapToElementsRecursive( text, mapping, spanHighlightIndices, wrapFunc, ); + + return output; } else { return text; } }; + +/** + * + * @param text + * @param spanHighlightIndices + * @param wrapFunc + * @returns Array of plaintext or components directly renderable by react + */ +export const wrapPlainTextSpan = (text, spanHighlightIndices, wrapFunc) => { + let outputSentences = []; + const textLength = text.length; + let currentIndex = 0; + + for (let i = 0; i < spanHighlightIndices.length; i++) { + const spanInfo = spanHighlightIndices[i]; + const spanIndexStart = spanInfo.indices[0]; + // Sometimes the end index is negative so we have to check this + const spanIndexEnd = + spanInfo.indices[1] > -1 ? spanInfo.indices[1] : textLength; + + // Append but don't highlight any span before + if (currentIndex < spanIndexStart) { + outputSentences.push(text.substring(currentIndex, spanIndexStart)); + } + + const spanText = text.substring(spanIndexStart, spanIndexEnd); + if (wrapFunc) { + outputSentences.push(wrapFunc(spanText, spanInfo)); + } else { + outputSentences.push(spanText); + } + + currentIndex = spanIndexEnd; + } + + // Append anything not highlighted + if (currentIndex < textLength) { + outputSentences.push(text.substring(currentIndex, textLength)); + } + + return outputSentences; +}; + +/** + * Classification variable is a map of categories where each one has a list of classified spans, we + * have to invert that so that we have a list of spans that contains all categories in that span + * @param filteredClassification + * @returns {*[]} + */ +export const mergeSpanIndices = (filteredClassification) => { + // classification variable is a map of categories where each one has a list of classified spans, we + // have to invert that so that we have a list of spans that contains all categories in that span + let mergedSpanIndices = []; + for (let label in filteredClassification) { + for (let i = 0; i < filteredClassification[label].length; i++) { + const spanInfo = filteredClassification[label][i]; + let matchingSpanFound = false; + + //Finds an existing matching span and append the technique, otherwise add the span to the list + for (let j = 0; j < mergedSpanIndices.length; j++) { + const mergedSpanInfo = mergedSpanIndices[j]; + if ( + spanInfo["indices"][0] === mergedSpanInfo["indices"][0] && + spanInfo["indices"][1] === mergedSpanInfo["indices"][1] + ) { + //Add technique to existing span + mergedSpanInfo["techniques"][label] = spanInfo["score"]; + matchingSpanFound = true; + } + } + if (!matchingSpanFound) { + mergedSpanIndices.push({ + indices: spanInfo["indices"], + techniques: { [label]: spanInfo["score"] }, + }); + } + } + } + + //Sort the spans by start index + mergedSpanIndices = _.orderBy(mergedSpanIndices, (obj) => { + return obj.indices[0]; + }); + + return mergedSpanIndices; +}; diff --git a/src/components/NavItems/tools/SemanticSearch/components/ResultDisplayItem.jsx b/src/components/NavItems/tools/SemanticSearch/components/ResultDisplayItem.jsx index f5d80b3b9..5c05e9a87 100644 --- a/src/components/NavItems/tools/SemanticSearch/components/ResultDisplayItem.jsx +++ b/src/components/NavItems/tools/SemanticSearch/components/ResultDisplayItem.jsx @@ -1,4 +1,5 @@ import React, { useState } from "react"; +import { useLocation } from "react-router-dom"; import { Avatar, Box, @@ -26,7 +27,13 @@ const ResultDisplayItem = ({ domainUrl, imageUrl, }) => { - const keyword = i18nLoadNamespace("components/NavItems/tools/SemanticSearch"); + const path = useLocation(); + let keyword; + if (path.pathname.includes("/app/assistant")) { + keyword = i18nLoadNamespace("components/NavItems/tools/Assistant"); + } else { + keyword = i18nLoadNamespace("components/NavItems/tools/SemanticSearch"); + } const [showOriginalClaim, setShowOriginalClaim] = useState(false); const [showOriginalTitle, setShowOriginalTitle] = useState(false); @@ -58,7 +65,7 @@ const ResultDisplayItem = ({ spacing={2} > - + {keyword("semantic_search_result_claim")}{" "} {showOriginalClaim ? claimOriginalLanguage : claim} @@ -81,7 +88,7 @@ const ResultDisplayItem = ({ )} - + {keyword("semantic_search_result_title")}{" "} {showOriginalTitle ? titleOriginalLanguage : title} diff --git a/src/components/NavItems/tools/SemanticSearch/index.jsx b/src/components/NavItems/tools/SemanticSearch/index.jsx index 83af9cbd2..53b6de4c7 100644 --- a/src/components/NavItems/tools/SemanticSearch/index.jsx +++ b/src/components/NavItems/tools/SemanticSearch/index.jsx @@ -1,5 +1,5 @@ import React, { useEffect, useState } from "react"; - +import { useParams } from "react-router-dom"; import { Alert, Backdrop, @@ -44,6 +44,10 @@ const SemanticSearch = () => { const currentLang = useSelector((state) => state.language); + const text = useSelector((state) => state.assistant.urlText); + + const { url } = useParams(); + /** * Helper function to return a list of language keys supported if the localized language is in duplicate keys * @param localizedLanguageName {string} the localized language to search keys for @@ -218,6 +222,17 @@ const SemanticSearch = () => { setLanguageFilter(loadLanguages); }, [currentLang]); + useEffect(() => { + //takes in text parameter from url + if (url) { + const uri = url !== null ? decodeURIComponent(url) : undefined; + if (uri === "assistantText" && text) { + setSearchString(text); + handleSubmit(); + } + } + }, [url]); + const handleSubmit = async () => { setIsLoading(true); setHasUserSubmittedForm(true); diff --git a/src/redux/actions/tools/assistantActions.jsx b/src/redux/actions/tools/assistantActions.jsx index 5c8f59886..eb1104982 100644 --- a/src/redux/actions/tools/assistantActions.jsx +++ b/src/redux/actions/tools/assistantActions.jsx @@ -163,6 +163,110 @@ export const setDbkfVideoMatchDetails = ( }; }; +export const setHpDetails = (hpResult, hpLoading, hpDone, hpFail) => { + return { + type: "SET_HP_DETAILS", + payload: { + hpResult: hpResult, + hpLoading: hpLoading, + hpDone: hpDone, + hpFail: hpFail, + }, + }; +}; + +export const setNewsTopicDetails = (ntResult, ntLoading, ntDone, ntFail) => { + return { + type: "SET_NEWS_TOPIC_DETAILS", + payload: { + newsFramingResult: ntResult, + newsFramingLoading: ntLoading, + newsFramingDone: ntDone, + newsFramingFail: ntFail, + }, + }; +}; + +export const setNewsGenreDetails = (ngResult, ngLoading, ngDone, ngFail) => { + return { + type: "SET_NEWS_GENRE_DETAILS", + payload: { + newsGenreResult: ngResult, + newsGenreLoading: ngLoading, + newsGenreDone: ngDone, + newsGenreFail: ngFail, + }, + }; +}; + +export const setPersuasionDetails = ( + perResult, + perLoading, + perDone, + perFail, +) => { + return { + type: "SET_PERSUASION_DETAILS", + payload: { + persuasionResult: perResult, + persuasionLoading: perLoading, + persuasionDone: perDone, + persuasionFail: perFail, + }, + }; +}; + +export const setSubjectivityDetails = ( + subResult, + subLoading, + subDone, + subFail, +) => { + return { + type: "SET_SUBJECTIVITY_DETAILS", + payload: { + subjectivityResult: subResult, + subjectivityLoading: subLoading, + subjectivityDone: subDone, + subjectivityFail: subFail, + }, + }; +}; + +export const setPrevFactChecksDetails = ( + pfcResult, + pfcLoading, + pfcDone, + pfcFail, +) => { + return { + type: "SET_PREV_FACT_CHECKS_DETAILS", + payload: { + prevFactChecksResult: pfcResult, + prevFactChecksLoading: pfcLoading, + prevFactChecksDone: pfcDone, + prevFactChecksFail: pfcFail, + }, + }; +}; + +export const setMachineGeneratedTextDetails = ( + mgtResult, + mgtLoading, + mgtDone, + mgtFail, +) => { + return { + type: "SET_MACHINE_GENERATED_TEXT_DETAILS", + payload: { + machineGeneratedTextResult: mgtResult, + machineGeneratedTextLoading: mgtLoading, + machineGeneratedTextDone: mgtDone, + machineGeneratedTextFail: mgtFail, + }, + }; +}; + export const setNeDetails = ( neResultCategory, neResultCount, diff --git a/src/redux/reducers/assistantReducer.jsx b/src/redux/reducers/assistantReducer.jsx index 55a0b70e1..caf1b8750 100644 --- a/src/redux/reducers/assistantReducer.jsx +++ b/src/redux/reducers/assistantReducer.jsx @@ -43,6 +43,36 @@ const defaultState = { neDone: false, neFail: false, + newsFramingResult: null, + newsFramingLoading: false, + newsFramingDone: false, + newsFramingFail: false, + + newsGenreResult: null, + newsGenreLoading: false, + newsGenreDone: false, + newsGenreFail: false, + + persuasionResult: null, + persuasionLoading: false, + persuasionDone: false, + persuasionFail: false, + + subjectivityResult: null, + subjectivityLoading: false, + subjectivityTextDone: false, + subjectivityTextFail: false, + + prevFactChecksResult: null, + prevFactChecksLoading: false, + prevFactChecksDone: false, + prevFactChecksFail: false, + + machineGeneratedTextResult: null, + machineGeneratedTextLoading: false, + machineGeneratedTextDone: false, + machineGeneratedTextFail: false, + loading: false, warningExpanded: false, assuranceExpanded: false, @@ -64,6 +94,12 @@ const assistantReducer = (state = defaultState, action) => { case "SET_DBKF_IMAGE_MATCH_DETAILS": case "SET_DBKF_VIDEO_MATCH_DETAILS": case "SET_NE_DETAILS": + case "SET_NEWS_TOPIC_DETAILS": + case "SET_NEWS_GENRE_DETAILS": + case "SET_PERSUASION_DETAILS": + case "SET_SUBJECTIVITY_DETAILS": + case "SET_PREV_FACT_CHECKS_DETAILS": + case "SET_MACHINE_GENERATED_TEXT_DETAILS": case "SET_LOADING": case "SET_WARNING_EXPANDED": case "SET_ASSURANCE_EXPANDED": diff --git a/src/redux/sagas/assistantSaga.jsx b/src/redux/sagas/assistantSaga.jsx index 4d25451b4..8a923ade3 100644 --- a/src/redux/sagas/assistantSaga.jsx +++ b/src/redux/sagas/assistantSaga.jsx @@ -12,6 +12,12 @@ import { setInputSourceCredDetails, setInputUrl, setNeDetails, + setNewsGenreDetails, + setNewsTopicDetails, + setPersuasionDetails, + setSubjectivityDetails, + setPrevFactChecksDetails, + setMachineGeneratedTextDetails, setProcessUrl, setProcessUrlActions, setScrapedData, @@ -76,6 +82,18 @@ function* getDbkfTextMatchSaga() { yield takeLatest(["SET_SCRAPED_DATA", "CLEAN_STATE"], handleDbkfTextCall); } +function* getNewsTopicSaga() { + yield takeLatest(["SET_SCRAPED_DATA", "CLEAN_STATE"], handleNewsTopicCall); +} + +function* getNewsGenreSaga() { + yield takeLatest(["SET_SCRAPED_DATA", "CLEAN_STATE"], handleNewsGenreCall); +} + +function* getPersuasionSaga() { + yield takeLatest(["SET_SCRAPED_DATA", "CLEAN_STATE"], handlePersuasionCall); +} + function* getSourceCredSaga() { yield takeLatest( ["SET_INPUT_URL", "CLEAN_STATE"], @@ -87,6 +105,24 @@ function* getNamedEntitySaga() { yield takeLatest(["SET_SCRAPED_DATA", "CLEAN_STATE"], handleNamedEntityCall); } +function* getSubjectivitySaga() { + yield takeLatest(["SET_SCRAPED_DATA", "CLEAN_STATE"], handleSubjectivityCall); +} + +function* getPrevFactChecksSaga() { + yield takeLatest( + ["SET_SCRAPED_DATA", "AUTH_USER_LOGIN", "CLEAN_STATE"], + handlePrevFactChecksCall, + ); +} + +function* getMachineGeneratedTextSaga() { + yield takeLatest( + ["SET_SCRAPED_DATA", "AUTH_USER_LOGIN", "CLEAN_STATE"], + handleMachineGeneratedTextCall, + ); +} + /** * NON-API HANDLERS **/ @@ -349,6 +385,124 @@ function* handleDbkfTextCall(action) { } } +function* handleNewsTopicCall(action) { + if (action.type === "CLEAN_STATE") return; + + try { + const text = yield select((state) => state.assistant.urlText); + + if (text) { + yield put(setNewsTopicDetails(null, true, false, false)); + + const result = yield call(assistantApi.callNewsFramingService, text); + yield put(setNewsTopicDetails(result, false, true, false)); + } + } catch (error) { + yield put(setNewsTopicDetails(null, false, false, true)); + } +} + +function* handleNewsGenreCall(action) { + if (action.type === "CLEAN_STATE") return; + + try { + const text = yield select((state) => state.assistant.urlText); + + if (text) { + yield put(setNewsGenreDetails(null, true, false, false)); + + const result = yield call(assistantApi.callNewsGenreService, text); + yield put(setNewsGenreDetails(result, false, true, false)); + } + } catch (error) { + yield put(setNewsGenreDetails(null, false, false, true)); + } +} + +function* handlePersuasionCall(action) { + if (action.type === "CLEAN_STATE") return; + + try { + const text = yield select((state) => state.assistant.urlText); + + if (text) { + yield put(setPersuasionDetails(null, true, false, false)); + + const result = yield call(assistantApi.callPersuasionService, text); + yield put(setPersuasionDetails(result, false, true, false)); + } + } catch (error) { + yield put(setPersuasionDetails(null, false, false, true)); + } +} + +function* handleSubjectivityCall(action) { + if (action.type === "CLEAN_STATE") return; + + try { + const text = yield select((state) => state.assistant.urlText); + + if (text) { + yield put(setSubjectivityDetails(null, true, false, false)); + + const result = yield call(assistantApi.callSubjectivityService, text); + + yield put(setSubjectivityDetails(result, false, true, false)); + } + } catch (error) { + yield put(setSubjectivityDetails(null, false, false, true)); + } +} + +function* handlePrevFactChecksCall(action) { + if (action.type === "CLEAN_STATE") return; + + try { + const text = yield select((state) => state.assistant.urlText); + + // this prevents the call from happening if not correct user status + const role = yield select((state) => state.userSession.user.roles); + + if (text && role.includes("BETA_TESTER")) { + yield put(setPrevFactChecksDetails(null, true, false, false)); + + const result = yield call(assistantApi.callPrevFactChecksService, text); + + yield put( + setPrevFactChecksDetails(result.fact_checks, false, true, false), + ); + } + } catch (error) { + yield put(setPrevFactChecksDetails(null, false, false, true)); + } +} + +function* handleMachineGeneratedTextCall(action) { + if (action.type === "CLEAN_STATE") return; + + try { + const text = yield select((state) => state.assistant.urlText); + + // this prevents the call from happening if not correct user status + + //yield take("SET_SCRAPED_DATA"); // wait until linkList has been created + const role = yield select((state) => state.userSession.user.roles); + + if (text && role.includes("BETA_TESTER")) { + yield put(setMachineGeneratedTextDetails(null, true, false, false)); + + const result = yield call( + assistantApi.callMachineGeneratedTextService, + text, + ); + + yield put(setMachineGeneratedTextDetails(result, false, true, false)); + } + } catch (error) { + yield put(setMachineGeneratedTextDetails(null, false, false, true)); + } +} + function* handleNamedEntityCall(action) { if (action.type === "CLEAN_STATE") return; @@ -689,7 +843,7 @@ const filterSourceCredibilityResults = ( linkList, trafficLightColors, ) => { - if (!originalResult.length) { + if (!originalResult) { return [null, null, null, null]; } let sourceCredibility = originalResult; @@ -868,5 +1022,11 @@ export default function* assistantSaga() { fork(getNamedEntitySaga), fork(getAssistantScrapeSaga), fork(getUploadSaga), + fork(getNewsTopicSaga), + fork(getNewsGenreSaga), + fork(getPersuasionSaga), + fork(getSubjectivitySaga), + fork(getPrevFactChecksSaga), + fork(getMachineGeneratedTextSaga), ]); }