tested with native dataset

yarenozyer · Jul 4, 2024 · 6a79a83 · 6a79a83
1 parent 60c784e
commit 6a79a83
Show file tree

Hide file tree

Showing 6 changed files with 22,354 additions and 67 deletions.
diff --git a/google_translate_train_aug_8_without_none.csv b/google_translate_train_aug_8_without_none.csv
diff --git a/gpt_train_without_none.csv b/gpt_train_without_none.csv
diff --git a/output.csv b/output.csv
diff --git a/parse_turkish_tweets.py b/parse_turkish_tweets.py
@@ -0,0 +1,17 @@
+import re
+
+def parse_turkish_tweets(text, target, stance, native_tweets, native_targets, native_stances):
+    cleaned_text = re.sub(r'<ENAMEX TYPE="[^"]+">([^<]+)</ENAMEX>', r'\1', text)
+
+    cleaned_text = re.sub(r'http\S+', '', cleaned_text)
+
+    cleaned_text = re.sub(r'@\S+', '', cleaned_text)
+    cleaned_text = re.sub(r'#\S+', '', cleaned_text)
+
+    sentences = re.split(r'\s*\n\s*', cleaned_text)
+    sentences = [sentence for sentence in sentences if sentence]
+    targets= [target for sentence in sentences if sentence]
+    stances = [stance for sentence in sentences if sentence]
+    native_tweets.extend(sentences)
+    native_targets.extend(targets)
+    native_stances.extend(stances)